From 3d74dc804611fb95832bb4e093f12c59238aff87 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Apr 2017 17:53:10 -0400 Subject: clean up after merge --- config.hs | 2 +- privdata/relocate | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 privdata/relocate diff --git a/config.hs b/config.hs index 97d90636..ec313725 120000 --- a/config.hs +++ b/config.hs @@ -1 +1 @@ -joeyconfig.hs \ No newline at end of file +config-simple.hs \ No newline at end of file diff --git a/privdata/relocate b/privdata/relocate deleted file mode 100644 index 271692d8..00000000 --- a/privdata/relocate +++ /dev/null @@ -1 +0,0 @@ -.joeyconfig -- cgit v1.2.3 From 983ee62929037c7297e2281ea3910e94a85bead5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 10 Apr 2017 10:52:33 -0400 Subject: reorg --- src/Propellor/Property/Bootstrap.hs | 39 ++----------------------------------- src/Propellor/Property/Chroot.hs | 33 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/Propellor/Property/Bootstrap.hs b/src/Propellor/Property/Bootstrap.hs index 5f64fd69..dc1c2e0f 100644 --- a/src/Propellor/Property/Bootstrap.hs +++ b/src/Propellor/Property/Bootstrap.hs @@ -5,12 +5,13 @@ import Propellor.Bootstrap import Propellor.Property.Chroot import Data.List -import System.Posix.Directory -- | Where a propellor repository should be bootstrapped from. data RepoSource = GitRepoUrl String | GitRepoOutsideChroot + -- ^ When used in a chroot, this clones the git repository from + -- outside the chroot. -- | Bootstraps a propellor installation into -- /usr/local/propellor/ @@ -38,10 +39,6 @@ bootstrappedFrom reposource = go `requires` clonedFrom reposource -- | Clones the propellor repeository into /usr/local/propellor/ -- --- GitRepoOutsideChroot can be used when this is used in a chroot. --- In that case, it clones the /usr/local/propellor/ from outside the --- chroot into the same path inside the chroot. --- -- If the propellor repo has already been cloned, pulls to get it -- up-to-date. clonedFrom :: RepoSource -> Property Linux @@ -82,38 +79,6 @@ clonedFrom reposource = property ("Propellor repo cloned from " ++ sourcedesc) $ GitRepoUrl s -> s GitRepoOutsideChroot -> localdir --- | Runs an action with the true localdir exposed, --- not the one bind-mounted into a chroot. The action is passed the --- path containing the contents of the localdir outside the chroot. --- --- In a chroot, this is accomplished by temporily bind mounting the localdir --- to a temp directory, to preserve access to the original bind mount. Then --- we unmount the localdir to expose the true localdir. Finally, to cleanup, --- the temp directory is bind mounted back to the localdir. -exposeTrueLocaldir :: (FilePath -> IO a) -> Propellor a -exposeTrueLocaldir a = ifM inChroot - ( liftIO $ withTmpDirIn (takeDirectory localdir) "propellor.tmp" $ \tmpdir -> - bracket_ - (movebindmount localdir tmpdir) - (movebindmount tmpdir localdir) - (a tmpdir) - , liftIO $ a localdir - ) - where - movebindmount from to = do - run "mount" [Param "--bind", File from, File to] - -- Have to lazy unmount, because the propellor process - -- is running in the localdir that it's unmounting.. - run "umount" [Param "-l", File from] - -- We were in the old localdir; move to the new one after - -- flipping the bind mounts. Otherwise, commands that try - -- to access the cwd will fail because it got umounted out - -- from under. - changeWorkingDirectory "/" - changeWorkingDirectory localdir - run cmd ps = unlessM (boolSystem cmd ps) $ - error $ "exposeTrueLocaldir failed to run " ++ show (cmd, ps) - assumeChange :: Propellor Bool -> Propellor Result assumeChange a = do ok <- a diff --git a/src/Propellor/Property/Chroot.hs b/src/Propellor/Property/Chroot.hs index 7738d97e..96c75846 100644 --- a/src/Propellor/Property/Chroot.hs +++ b/src/Propellor/Property/Chroot.hs @@ -11,6 +11,7 @@ module Propellor.Property.Chroot ( ChrootTarball(..), noServices, inChroot, + exposeTrueLocaldir, -- * Internal use provisioned', propagateChrootInfo, @@ -295,6 +296,38 @@ setInChroot h = h { hostInfo = hostInfo h `addInfo` InfoVal (InChroot True) } newtype InChroot = InChroot Bool deriving (Typeable, Show) +-- | Runs an action with the true localdir exposed, +-- not the one bind-mounted into a chroot. The action is passed the +-- path containing the contents of the localdir outside the chroot. +-- +-- In a chroot, this is accomplished by temporily bind mounting the localdir +-- to a temp directory, to preserve access to the original bind mount. Then +-- we unmount the localdir to expose the true localdir. Finally, to cleanup, +-- the temp directory is bind mounted back to the localdir. +exposeTrueLocaldir :: (FilePath -> IO a) -> Propellor a +exposeTrueLocaldir a = ifM inChroot + ( liftIO $ withTmpDirIn (takeDirectory localdir) "propellor.tmp" $ \tmpdir -> + bracket_ + (movebindmount localdir tmpdir) + (movebindmount tmpdir localdir) + (a tmpdir) + , liftIO $ a localdir + ) + where + movebindmount from to = do + run "mount" [Param "--bind", File from, File to] + -- Have to lazy unmount, because the propellor process + -- is running in the localdir that it's unmounting.. + run "umount" [Param "-l", File from] + -- We were in the old localdir; move to the new one after + -- flipping the bind mounts. Otherwise, commands that try + -- to access the cwd will fail because it got umounted out + -- from under. + changeWorkingDirectory "/" + changeWorkingDirectory localdir + run cmd ps = unlessM (boolSystem cmd ps) $ + error $ "exposeTrueLocaldir failed to run " ++ show (cmd, ps) + -- | Generates a Chroot that has all the properties of a Host. -- -- Note that it's possible to create loops using this, where a host -- cgit v1.2.3 From 03950541b77405b8822dd2cadb47bc249a2bb5d3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 10 Apr 2017 11:12:17 -0400 Subject: copy git configuration into chroot --- src/Propellor/Property/Bootstrap.hs | 82 +++++++++++++++++++++++-------------- src/Propellor/Property/Chroot.hs | 8 ++-- 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/Propellor/Property/Bootstrap.hs b/src/Propellor/Property/Bootstrap.hs index dc1c2e0f..5678a865 100644 --- a/src/Propellor/Property/Bootstrap.hs +++ b/src/Propellor/Property/Bootstrap.hs @@ -5,13 +5,14 @@ import Propellor.Bootstrap import Propellor.Property.Chroot import Data.List +import qualified Data.ByteString as B -- | Where a propellor repository should be bootstrapped from. data RepoSource = GitRepoUrl String | GitRepoOutsideChroot - -- ^ When used in a chroot, this clones the git repository from - -- outside the chroot. + -- ^ When used in a chroot, this copies the git repository from + -- outside the chroot, including its configuration. -- | Bootstraps a propellor installation into -- /usr/local/propellor/ @@ -42,42 +43,61 @@ bootstrappedFrom reposource = go `requires` clonedFrom reposource -- If the propellor repo has already been cloned, pulls to get it -- up-to-date. clonedFrom :: RepoSource -> Property Linux -clonedFrom reposource = property ("Propellor repo cloned from " ++ sourcedesc) $ do - ifM needclone - ( do - let tmpclone = localdir ++ ".tmpclone" - system <- getOS - assumeChange $ exposeTrueLocaldir $ \sysdir -> do - let originloc = case reposource of - GitRepoUrl s -> s - GitRepoOutsideChroot -> sysdir - runShellCommand $ buildShellCommand - [ installGitCommand system - , "rm -rf " ++ tmpclone - , "git clone " ++ shellEscape originloc ++ " " ++ tmpclone - , "mkdir -p " ++ localdir - -- This is done rather than deleting - -- the old localdir, because if it is bound - -- mounted from outside the chroot, deleting - -- it after unmounting in unshare will remove - -- the bind mount outside the unshare. - , "(cd " ++ tmpclone ++ " && tar c .) | (cd " ++ localdir ++ " && tar x)" - , "rm -rf " ++ tmpclone - ] - , assumeChange $ exposeTrueLocaldir $ const $ +clonedFrom reposource = case reposource of + GitRepoOutsideChroot -> go `onChange` copygitconfig + _ -> go + where + go :: Property Linux + go = property ("Propellor repo cloned from " ++ sourcedesc) $ + ifM needclone (makeclone, updateclone) + + makeclone = do + let tmpclone = localdir ++ ".tmpclone" + system <- getOS + assumeChange $ exposeTrueLocaldir $ \sysdir -> do + let originloc = case reposource of + GitRepoUrl s -> s + GitRepoOutsideChroot -> sysdir runShellCommand $ buildShellCommand - [ "cd " ++ localdir - , "git pull" + [ installGitCommand system + , "rm -rf " ++ tmpclone + , "git clone " ++ shellEscape originloc ++ " " ++ tmpclone + , "mkdir -p " ++ localdir + -- This is done rather than deleting + -- the old localdir, because if it is bound + -- mounted from outside the chroot, deleting + -- it after unmounting in unshare will remove + -- the bind mount outside the unshare. + , "(cd " ++ tmpclone ++ " && tar c .) | (cd " ++ localdir ++ " && tar x)" + , "rm -rf " ++ tmpclone ] - ) - where + + updateclone = assumeChange $ exposeTrueLocaldir $ const $ + runShellCommand $ buildShellCommand + [ "cd " ++ localdir + , "git pull" + ] + + -- Copy the git config of the repo outside the chroot into the + -- chroot. This way it has the same remote urls, and other git + -- configuration. + copygitconfig :: Property Linux + copygitconfig = property ("Propellor repo git config copied from outside the chroot") $ do + let gitconfig = localdir <> ".git" <> "config" + cfg <- liftIO $ B.readFile gitconfig + exposeTrueLocaldir $ const $ + liftIO $ B.writeFile gitconfig cfg + return MadeChange + needclone = (inChroot <&&> truelocaldirisempty) <||> (liftIO (not <$> doesDirectoryExist localdir)) + truelocaldirisempty = exposeTrueLocaldir $ const $ runShellCommand ("test ! -d " ++ localdir ++ "/.git") + sourcedesc = case reposource of GitRepoUrl s -> s - GitRepoOutsideChroot -> localdir + GitRepoOutsideChroot -> localdir ++ " outside the chroot" assumeChange :: Propellor Bool -> Propellor Result assumeChange a = do @@ -87,5 +107,5 @@ assumeChange a = do buildShellCommand :: [String] -> String buildShellCommand = intercalate "&&" . map (\c -> "(" ++ c ++ ")") -runShellCommand :: String -> IO Bool +runShellCommand :: String -> Propellor Bool runShellCommand s = liftIO $ boolSystem "sh" [ Param "-c", Param s] diff --git a/src/Propellor/Property/Chroot.hs b/src/Propellor/Property/Chroot.hs index 96c75846..5f764d47 100644 --- a/src/Propellor/Property/Chroot.hs +++ b/src/Propellor/Property/Chroot.hs @@ -304,17 +304,17 @@ newtype InChroot = InChroot Bool -- to a temp directory, to preserve access to the original bind mount. Then -- we unmount the localdir to expose the true localdir. Finally, to cleanup, -- the temp directory is bind mounted back to the localdir. -exposeTrueLocaldir :: (FilePath -> IO a) -> Propellor a +exposeTrueLocaldir :: (FilePath -> Propellor a) -> Propellor a exposeTrueLocaldir a = ifM inChroot - ( liftIO $ withTmpDirIn (takeDirectory localdir) "propellor.tmp" $ \tmpdir -> + ( withTmpDirIn (takeDirectory localdir) "propellor.tmp" $ \tmpdir -> bracket_ (movebindmount localdir tmpdir) (movebindmount tmpdir localdir) (a tmpdir) - , liftIO $ a localdir + , a localdir ) where - movebindmount from to = do + movebindmount from to = liftIO $ do run "mount" [Param "--bind", File from, File to] -- Have to lazy unmount, because the propellor process -- is running in the localdir that it's unmounting.. -- cgit v1.2.3 From 3df52cf2fb5cdb92b407195f55467e3b5ef26785 Mon Sep 17 00:00:00 2001 From: joelmccracken Date: Thu, 13 Apr 2017 20:50:08 +0000 Subject: --- doc/forum/Work_on_OS_X.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/Work_on_OS_X.mdwn diff --git a/doc/forum/Work_on_OS_X.mdwn b/doc/forum/Work_on_OS_X.mdwn new file mode 100644 index 00000000..e3c5fd64 --- /dev/null +++ b/doc/forum/Work_on_OS_X.mdwn @@ -0,0 +1,5 @@ +I'm interested in using Propellor on OS X. I understand that it is not supported though. + +Is there anyone doing this? If it was developed, would support for OS X be merged upstream? + +Thanks! -- cgit v1.2.3 From 7dbde4c109b09eb8eee198e71de1500846fc7a88 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 13 Apr 2017 17:42:52 -0400 Subject: response --- ...mment_1_6d7d5b89f1de9604718f7973e4b3eeb1._comment | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/Work_on_OS_X/comment_1_6d7d5b89f1de9604718f7973e4b3eeb1._comment diff --git a/doc/forum/Work_on_OS_X/comment_1_6d7d5b89f1de9604718f7973e4b3eeb1._comment b/doc/forum/Work_on_OS_X/comment_1_6d7d5b89f1de9604718f7973e4b3eeb1._comment new file mode 100644 index 00000000..4eac2063 --- /dev/null +++ b/doc/forum/Work_on_OS_X/comment_1_6d7d5b89f1de9604718f7973e4b3eeb1._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-04-13T21:36:20Z" + content=""" +I got a patch some years back to make propellor compile on OSX. +I merged it. You might want to get in touch with its author, as +he may be doing something with propellor on OSX. + + +Anyway, I'd probably merge OSX patches, if they were not super +intrusive. And I don't see why it would be, as propellor already supports +FreeBSD. + +Since `Property` is parameterized by the operating systems it +supports, it should be easy to start by only porting the core parts +of propellor, and then port over individual Properties one by one as +needed. See the commits for the recent FreeBSD port for a nice walkthough +of the changes you'll want to make. +"""]] -- cgit v1.2.3 From b7622d7e2391623a99a899c31b8edf960ade6dfc Mon Sep 17 00:00:00 2001 From: joelmccracken Date: Mon, 17 Apr 2017 17:47:30 +0000 Subject: Added a comment --- .../comment_2_00b20c240fc13bed6dc54e5b985b41e2._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/forum/Work_on_OS_X/comment_2_00b20c240fc13bed6dc54e5b985b41e2._comment diff --git a/doc/forum/Work_on_OS_X/comment_2_00b20c240fc13bed6dc54e5b985b41e2._comment b/doc/forum/Work_on_OS_X/comment_2_00b20c240fc13bed6dc54e5b985b41e2._comment new file mode 100644 index 00000000..aa33c85b --- /dev/null +++ b/doc/forum/Work_on_OS_X/comment_2_00b20c240fc13bed6dc54e5b985b41e2._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joelmccracken" + avatar="http://cdn.libravatar.org/avatar/45175015b9eb3dd3f6c740b3fe920fed" + subject="comment 2" + date="2017-04-17T17:47:30Z" + content=""" +Sounds good. I contacted the person you linked to, have not heard back yet. + + + +The first issue I ran into is that propellor wants to connect to \"root@\", and it doesn't look like this is configurable. +Would you accept a patch to make this configurable? + +Additionally, is this the best place to ask questions about what you would/would not accept? + +Thank you!!! +"""]] -- cgit v1.2.3 From 162b0d77dd607316b79dad2aa7d0e2986927746b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 17 Apr 2017 20:12:12 -0400 Subject: response --- .../comment_3_294f4783522a8e4887793aac921ee546._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/Work_on_OS_X/comment_3_294f4783522a8e4887793aac921ee546._comment diff --git a/doc/forum/Work_on_OS_X/comment_3_294f4783522a8e4887793aac921ee546._comment b/doc/forum/Work_on_OS_X/comment_3_294f4783522a8e4887793aac921ee546._comment new file mode 100644 index 00000000..ed654d3f --- /dev/null +++ b/doc/forum/Work_on_OS_X/comment_3_294f4783522a8e4887793aac921ee546._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-04-18T00:08:13Z" + content=""" +Yes, this is the place. Or you can email me directly, but I prefer to keep +discussions public. + +`propellor --spin` needs a way to run commands as root on the remote host. +If ssh as root on OSX is not allowed, it would need a way to get to a user +who can get root, and it would be very annoying if a password needed to be +entered since each `propellor --spin` actually makes several ssh connections to +the remote host. Anything that works within these constraints would be ok. +"""]] -- cgit v1.2.3 From 743d0a7b0fe0fae3dc3fc65307c33a2b2da82521 Mon Sep 17 00:00:00 2001 From: joelmccracken Date: Thu, 20 Apr 2017 02:23:06 +0000 Subject: Added a comment --- .../comment_4_74b579d4d590432b6bd236ccb929cc11._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/forum/Work_on_OS_X/comment_4_74b579d4d590432b6bd236ccb929cc11._comment diff --git a/doc/forum/Work_on_OS_X/comment_4_74b579d4d590432b6bd236ccb929cc11._comment b/doc/forum/Work_on_OS_X/comment_4_74b579d4d590432b6bd236ccb929cc11._comment new file mode 100644 index 00000000..d386c1b5 --- /dev/null +++ b/doc/forum/Work_on_OS_X/comment_4_74b579d4d590432b6bd236ccb929cc11._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joelmccracken" + avatar="http://cdn.libravatar.org/avatar/45175015b9eb3dd3f6c740b3fe920fed" + subject="comment 4" + date="2017-04-20T02:23:06Z" + content=""" +So, it turns out that yes, root is a thing on os x... but it is complicated. I'm going to put what I learned here because I think it will be useful, at least for telling folks how to use propellor on os x. + +1. Enable the root account. Steps are here: https://support.apple.com/en-us/HT204012 +2. password-authentication as root is disabled -- if you try to `ssh root@localhost`, it wont work. you need a key pair. +3. use su/sudo to install a public key (probably at `.ssh/id_rsa.pub`) to roots authorized_keys. adapted from: https://discussions.apple.com/thread/4078360?start=0&tstart=0 +4. copy the the pub file to authorized keys: `sudo cp /Users/joel/.ssh/id_rsa.pub /var/root/.ssh/authorized_keys` +5. you should now be able to `ssh root@localhost` without a password. + +I'm not super sure that this is even the best way forward, but lets get this working first, then we'll see. +"""]] -- cgit v1.2.3 From c72d3f8fc88691572cb4531ea1760784bca0661d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Apr 2017 00:57:56 -0400 Subject: releasing package propellor version 4.0.3 --- debian/changelog | 4 ++-- propellor.cabal | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 70aa139d..498cb5bc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,11 +1,11 @@ -propellor (4.0.3) UNRELEASED; urgency=medium +propellor (4.0.3) unstable; urgency=medium * Added Fstab.listed, Fstab.swap, and Mount.swapOn properties. Thanks, Daniel Brooks. * Added Propellor.Property.Bootstrap, which can be used to make disk images contain their own installation of propellor. - -- Joey Hess Thu, 06 Apr 2017 19:40:12 -0400 + -- Joey Hess Thu, 20 Apr 2017 00:54:32 -0400 propellor (4.0.2) unstable; urgency=medium diff --git a/propellor.cabal b/propellor.cabal index 714c3235..58651a01 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -1,5 +1,5 @@ Name: propellor -Version: 4.0.2 +Version: 4.0.3 Cabal-Version: >= 1.8 License: BSD2 Maintainer: Joey Hess -- cgit v1.2.3 From da9e56615f09583d94edf78d72455ec114be866d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Apr 2017 00:58:21 -0400 Subject: add news item for propellor 4.0.3 --- doc/news/version_4.0.2.mdwn | 12 ------------ doc/news/version_4.0.3.mdwn | 6 ++++++ 2 files changed, 6 insertions(+), 12 deletions(-) delete mode 100644 doc/news/version_4.0.2.mdwn create mode 100644 doc/news/version_4.0.3.mdwn diff --git a/doc/news/version_4.0.2.mdwn b/doc/news/version_4.0.2.mdwn deleted file mode 100644 index b955c579..00000000 --- a/doc/news/version_4.0.2.mdwn +++ /dev/null @@ -1,12 +0,0 @@ -propellor 4.0.2 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Apt.mirror can be used to set the preferred apt mirror of a host, - overriding the default CDN. This info is used by - Apt.stdSourcesList and Sbuild.builtFor. - Thanks, Sean Whitton. - * Property.Partition: Update kpartx output parser, as its output format - changed around version 0.6. Both output formats are supported now. - * Fix bug when using setContainerProps with a chroot that prevented - properties added to a chroot that way from being seen when propellor - was running inside the chroot. This affected disk image creation, and - possibly other things that use chroots."""]] \ No newline at end of file diff --git a/doc/news/version_4.0.3.mdwn b/doc/news/version_4.0.3.mdwn new file mode 100644 index 00000000..eb467287 --- /dev/null +++ b/doc/news/version_4.0.3.mdwn @@ -0,0 +1,6 @@ +propellor 4.0.3 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Added Fstab.listed, Fstab.swap, and Mount.swapOn properties. + Thanks, Daniel Brooks. + * Added Propellor.Property.Bootstrap, which can be used to make + disk images contain their own installation of propellor."""]] \ No newline at end of file -- cgit v1.2.3 From 33890da9cb62c6d621b8fb4db69af2eb6810e1f4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Apr 2017 19:35:54 -0400 Subject: increase cabal-version f045116b618e255c583376447be635c245d63909 does not work with cabal 1.16 --- propellor.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/propellor.cabal b/propellor.cabal index 58651a01..e217bfd7 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -1,6 +1,6 @@ Name: propellor Version: 4.0.3 -Cabal-Version: >= 1.8 +Cabal-Version: >= 1.20 License: BSD2 Maintainer: Joey Hess Author: Joey Hess -- cgit v1.2.3 From fe4b58f7db06cd59b95e73ef2a664372d0a4addd Mon Sep 17 00:00:00 2001 From: Félix Sipma Date: Thu, 27 Apr 2017 14:57:12 +0200 Subject: add Restic module --- propellor.cabal | 1 + src/Propellor/Property/Restic.hs | 204 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 src/Propellor/Property/Restic.hs diff --git a/propellor.cabal b/propellor.cabal index 58651a01..292bc79d 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -136,6 +136,7 @@ Library Propellor.Property.PropellorRepo Propellor.Property.Prosody Propellor.Property.Reboot + Propellor.Property.Restic Propellor.Property.Rsync Propellor.Property.Sbuild Propellor.Property.Scheduled diff --git a/src/Propellor/Property/Restic.hs b/src/Propellor/Property/Restic.hs new file mode 100644 index 00000000..55a68324 --- /dev/null +++ b/src/Propellor/Property/Restic.hs @@ -0,0 +1,204 @@ +-- | Maintainer: Félix Sipma +-- +-- Support for the restic backup tool + +module Propellor.Property.Restic + ( ResticRepo (..) + , installed + , repoExists + , init + , restored + , backup + , KeepPolicy (..) + ) where + +import Propellor.Base hiding (init) +import Prelude hiding (init) +import qualified Propellor.Property.Apt as Apt +import qualified Propellor.Property.Cron as Cron +import qualified Propellor.Property.File as File +import Data.List (intercalate) + +type Url = String + +type ResticParam = String + +data ResticRepo + = Direct FilePath + | SFTP User HostName FilePath + | REST Url + +instance ConfigurableValue ResticRepo where + val (Direct fp) = fp + val (SFTP u h fp) = "sftp:" ++ val u ++ "@" ++ val h ++ ":" ++ fp + val (REST url) = "rest:" ++ url + +installed :: Property DebianLike +installed = withOS desc $ \w o -> case o of + (Just (System (Debian _ (Stable "jessie")) _)) -> ensureProperty w $ + Apt.installedBackport ["restic"] + _ -> ensureProperty w $ + Apt.installed ["restic"] + where + desc = "installed restic" + +repoExists :: ResticRepo -> IO Bool +repoExists repo = boolSystem "restic" + [ Param "-r" + , File (val repo) + , Param "--password-file" + , File (getPasswordFile repo) + , Param "snapshots" + ] + +passwordFileDir :: FilePath +passwordFileDir = "/etc/restic-keys" + +getPasswordFile :: ResticRepo -> FilePath +getPasswordFile repo = passwordFileDir File.configFileName (val repo) + +passwordFileConfigured :: ResticRepo -> Property (HasInfo + UnixLike) +passwordFileConfigured repo = propertyList "restic password file" $ props + & File.dirExists passwordFileDir + & File.mode passwordFileDir 0O2700 + & getPasswordFile repo `File.hasPrivContent` hostContext + +-- | Inits a new restic repository +init :: ResticRepo -> Property (HasInfo + DebianLike) +init repo = check (not <$> repoExists repo) (cmdProperty "restic" initargs) + `requires` installed + `requires` passwordFileConfigured repo + where + initargs = + [ "-r" + , val repo + , "--password-file" + , getPasswordFile repo + , "init" + ] + +-- | Restores a directory from a restic backup. +-- +-- Only does anything if the directory does not exist, or exists, +-- but is completely empty. +-- +-- The restore is performed atomically; restoring to a temp directory +-- and then moving it to the directory. +restored :: FilePath -> ResticRepo -> Property (HasInfo + DebianLike) +restored dir repo = go + `requires` installed + `requires` passwordFileConfigured repo + where + go :: Property DebianLike + go = property (dir ++ " restored by restic") $ ifM (liftIO needsRestore) + ( do + warningMessage $ dir ++ " is empty/missing; restoring from backup ..." + liftIO restore + , noChange + ) + + needsRestore = null <$> catchDefaultIO [] (dirContents dir) + + restore = withTmpDirIn (takeDirectory dir) "restic-restore" $ \tmpdir -> do + ok <- boolSystem "restic" + [ Param "-r" + , File (val repo) + , Param "--password-file" + , File (getPasswordFile repo) + , Param "restore" + , Param "latest" + , Param "--target" + , File tmpdir + ] + let restoreddir = tmpdir ++ "/" ++ dir + ifM (pure ok <&&> doesDirectoryExist restoreddir) + ( do + void $ tryIO $ removeDirectory dir + renameDirectory restoreddir dir + return MadeChange + , return FailedChange + ) + +-- | Installs a cron job that causes a given directory to be backed +-- up, by running borg with some parameters. +-- +-- If the directory does not exist, or exists but is completely empty, +-- this Property will immediately restore it from an existing backup. +-- +-- So, this property can be used to deploy a directory of content +-- to a host, while also ensuring any changes made to it get backed up. +-- For example: +-- +-- > & Restic.backup "/srv/git" +-- > (Restic.SFTP (User root) (HostName myserver) /mnt/backup/git.restic") +-- > Cron.Daily +-- > ["--exclude=/srv/git/tobeignored"] +-- > [Restic.KeepDays 7, Restic.KeepWeeks 4, Restic.KeepMonths 6, Restic.KeepYears 1] +-- +-- Since restic uses a fair amount of system resources, only one restic +-- backup job will be run at a time. Other jobs will wait their turns to +-- run. +backup :: FilePath -> ResticRepo -> Cron.Times -> [ResticParam] -> [KeepPolicy] -> Property (HasInfo + DebianLike) +backup dir repo crontimes extraargs kp = backup' dir repo crontimes extraargs kp + `requires` restored dir repo + +-- | Does a backup, but does not automatically restore. +backup' :: FilePath -> ResticRepo -> Cron.Times -> [ResticParam] -> [KeepPolicy] -> Property (HasInfo + DebianLike) +backup' dir repo crontimes extraargs kp = cronjob + `describe` desc + `requires` installed + `requires` passwordFileConfigured repo + where + desc = val repo ++ " restic backup" + cronjob = Cron.niceJob ("restic_backup" ++ dir) crontimes (User "root") "/" $ + "flock " ++ shellEscape lockfile ++ " sh -c " ++ backupcmd + lockfile = "/var/lock/propellor-restic.lock" + backupcmd = intercalate ";" $ + createCommand + : if null kp then [] else [pruneCommand] + createCommand = unwords $ + [ "restic" + , "-r" + , val repo + , "--password-file" + , getPasswordFile repo + ] + ++ map shellEscape extraargs ++ + [ "backup" + , shellEscape dir + ] + pruneCommand = unwords $ + [ "restic" + , "-r" + , val repo + , "--password-file" + , getPasswordFile repo + , "forget" + , "--prune" + ] + ++ + map keepParam kp + +-- | Constructs a ResticParam that specifies which old backup generations to +-- keep. By default, all generations are kept. However, when this parameter is +-- passed to the `backup` property, they will run restic prune to clean out +-- generations not specified here. +keepParam :: KeepPolicy -> ResticParam +keepParam (KeepLast n) = "--keep-last=" ++ val n +keepParam (KeepHours n) = "--keep-hourly=" ++ val n +keepParam (KeepDays n) = "--keep-daily=" ++ val n +keepParam (KeepWeeks n) = "--keep-weekly=" ++ val n +keepParam (KeepMonths n) = "--keep-monthly=" ++ val n +keepParam (KeepYears n) = "--keep-yearly=" ++ val n + +-- | Policy for backup generations to keep. For example, KeepDays 30 will +-- keep the latest backup for each day when a backup was made, and keep the +-- last 30 such backups. When multiple KeepPolicies are combined together, +-- backups meeting any policy are kept. See borg's man page for details. +data KeepPolicy + = KeepLast Int + | KeepHours Int + | KeepDays Int + | KeepWeeks Int + | KeepMonths Int + | KeepYears Int -- cgit v1.2.3 From f6b2ab29f24c7399ed7ab718c541eb46bc0f24f7 Mon Sep 17 00:00:00 2001 From: Félix Sipma Date: Thu, 27 Apr 2017 19:17:34 +0200 Subject: Restic: make sure the repo exists before running restic commands --- src/Propellor/Property/Restic.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Propellor/Property/Restic.hs b/src/Propellor/Property/Restic.hs index 55a68324..668843bb 100644 --- a/src/Propellor/Property/Restic.hs +++ b/src/Propellor/Property/Restic.hs @@ -86,8 +86,7 @@ init repo = check (not <$> repoExists repo) (cmdProperty "restic" initargs) -- and then moving it to the directory. restored :: FilePath -> ResticRepo -> Property (HasInfo + DebianLike) restored dir repo = go - `requires` installed - `requires` passwordFileConfigured repo + `requires` init repo where go :: Property DebianLike go = property (dir ++ " restored by restic") $ ifM (liftIO needsRestore) @@ -146,8 +145,7 @@ backup dir repo crontimes extraargs kp = backup' dir repo crontimes extraargs kp backup' :: FilePath -> ResticRepo -> Cron.Times -> [ResticParam] -> [KeepPolicy] -> Property (HasInfo + DebianLike) backup' dir repo crontimes extraargs kp = cronjob `describe` desc - `requires` installed - `requires` passwordFileConfigured repo + `requires` init repo where desc = val repo ++ " restic backup" cronjob = Cron.niceJob ("restic_backup" ++ dir) crontimes (User "root") "/" $ -- cgit v1.2.3 From 1b7abb5d209e4bdb66737f7fbdbc312e7802f081 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 27 Apr 2017 16:31:18 -0400 Subject: few little things --- src/Propellor/Property/Restic.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Propellor/Property/Restic.hs b/src/Propellor/Property/Restic.hs index 668843bb..ef867de3 100644 --- a/src/Propellor/Property/Restic.hs +++ b/src/Propellor/Property/Restic.hs @@ -40,7 +40,7 @@ installed = withOS desc $ \w o -> case o of _ -> ensureProperty w $ Apt.installed ["restic"] where - desc = "installed restic" + desc = "installed restic" repoExists :: ResticRepo -> IO Bool repoExists repo = boolSystem "restic" @@ -119,7 +119,7 @@ restored dir repo = go ) -- | Installs a cron job that causes a given directory to be backed --- up, by running borg with some parameters. +-- up, by running restic with some parameters. -- -- If the directory does not exist, or exists but is completely empty, -- this Property will immediately restore it from an existing backup. @@ -192,7 +192,7 @@ keepParam (KeepYears n) = "--keep-yearly=" ++ val n -- | Policy for backup generations to keep. For example, KeepDays 30 will -- keep the latest backup for each day when a backup was made, and keep the -- last 30 such backups. When multiple KeepPolicies are combined together, --- backups meeting any policy are kept. See borg's man page for details. +-- backups meeting any policy are kept. See restic's man page for details. data KeepPolicy = KeepLast Int | KeepHours Int -- cgit v1.2.3 From fabce00bc20a176fb593f6107de5c54f45cf1896 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 27 Apr 2017 16:32:09 -0400 Subject: Propellor.Property.Restic added for yet another backup program. Thanks, Félix Sipma. --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 498cb5bc..43e3cd6f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +propellor (4.0.4) UNRELEASED; urgency=medium + + * Propellor.Property.Restic added for yet another backup program. + Thanks, Félix Sipma. + + -- Joey Hess Thu, 27 Apr 2017 16:31:26 -0400 + propellor (4.0.3) unstable; urgency=medium * Added Fstab.listed, Fstab.swap, and Mount.swapOn properties. -- cgit v1.2.3 From b06edbda0478ed57954d716f64f6870d7ae68f63 Mon Sep 17 00:00:00 2001 From: Félix Sipma Date: Fri, 28 Apr 2017 00:19:46 +0200 Subject: Restic: fix bug in shell escaping --- src/Propellor/Property/Restic.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Propellor/Property/Restic.hs b/src/Propellor/Property/Restic.hs index 668843bb..02b2ead0 100644 --- a/src/Propellor/Property/Restic.hs +++ b/src/Propellor/Property/Restic.hs @@ -149,17 +149,17 @@ backup' dir repo crontimes extraargs kp = cronjob where desc = val repo ++ " restic backup" cronjob = Cron.niceJob ("restic_backup" ++ dir) crontimes (User "root") "/" $ - "flock " ++ shellEscape lockfile ++ " sh -c " ++ backupcmd + "flock " ++ shellEscape lockfile ++ " sh -c " ++ shellEscape backupcmd lockfile = "/var/lock/propellor-restic.lock" - backupcmd = intercalate ";" $ + backupcmd = intercalate " && " $ createCommand : if null kp then [] else [pruneCommand] createCommand = unwords $ [ "restic" , "-r" - , val repo + , shellEscape (val repo) , "--password-file" - , getPasswordFile repo + , shellEscape (getPasswordFile repo) ] ++ map shellEscape extraargs ++ [ "backup" @@ -168,9 +168,9 @@ backup' dir repo crontimes extraargs kp = cronjob pruneCommand = unwords $ [ "restic" , "-r" - , val repo + , shellEscape (val repo) , "--password-file" - , getPasswordFile repo + , shellEscape (getPasswordFile repo) , "forget" , "--prune" ] -- cgit v1.2.3 From 89d216c838f989bcc8783e4b3a650a2157286389 Mon Sep 17 00:00:00 2001 From: bardur.arantsson Date: Fri, 12 May 2017 06:50:49 +0000 Subject: Added a comment --- ...ent_3_cd4b9b9e160469e9f0b105f6c40a4ef8._comment | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 doc/forum/Using_propellor_for_continers_only/comment_3_cd4b9b9e160469e9f0b105f6c40a4ef8._comment diff --git a/doc/forum/Using_propellor_for_continers_only/comment_3_cd4b9b9e160469e9f0b105f6c40a4ef8._comment b/doc/forum/Using_propellor_for_continers_only/comment_3_cd4b9b9e160469e9f0b105f6c40a4ef8._comment new file mode 100644 index 00000000..fceeedcf --- /dev/null +++ b/doc/forum/Using_propellor_for_continers_only/comment_3_cd4b9b9e160469e9f0b105f6c40a4ef8._comment @@ -0,0 +1,54 @@ +[[!comment format=mdwn + username="bardur.arantsson" + avatar="http://cdn.libravatar.org/avatar/a0be0039b44d33262b7ae650a0803ad5" + subject="comment 3" + date="2017-05-12T06:50:49Z" + content=""" +Ok, so I've tried to use this to build a Chroot (a reasonable starting point for building containers), using the following program: + + module Main + ( main + ) where + + import Propellor + import Propellor.Engine + import Propellor.Property.DiskImage + import qualified Propellor.Property.Apt as Apt + import qualified Propellor.Property.User as User + import Propellor.Property.Chroot + + main :: IO () + main = mainProperties $ host \"whatever\" $ props + & provisioned (mychroot \"out\") + where + mychroot d = debootstrapped mempty d $ props + & osDebian Unstable X86_64 + & Apt.installed [\"linux-image-amd64\"] + & User.hasPassword (User \"root\") + & User.accountFor (User \"demo\") + & User.hasPassword (User \"demo\") + +It seems that \"debootstrap\" finishes: + + I: Configuring apt-transport-https... + I: Configuring tasksel... + I: Configuring tasksel-data... + I: Configuring libc-bin... + I: Configuring systemd... + I: Configuring ca-certificates... + I: Base system installed successfully. + +But fails immediately afterwards: + + ldd: /usr/local/propellor/propellor: No such file or directory + ** warning: user error (ldd [\"/usr/local/propellor/propellor\"] exited 1) + whatever chroot out exists ... failed + whatever overall ... failed + +(I should probably have used a different hostname than \"whatever\", but... whatever :).) + +So it seems that the chroot support still expects propellor to be installed on the host system? + +I should mention that I've done an extremely small patch to Propellor locally, just to the ChrootBootstrapper instance for ArchLinux to allow it to call debootstrap on Arch Linux -- it seems to exist as a package these days, not sure if it did when that Propellor code was written. Anyway... + +"""]] -- cgit v1.2.3 From 51b74d0397d5f6498f167c7820fb4966be2be7c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 May 2017 13:53:18 -0400 Subject: response --- ...ment_4_9dc985b26c29b9ce21e6c75ec03f6262._comment | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/forum/Using_propellor_for_continers_only/comment_4_9dc985b26c29b9ce21e6c75ec03f6262._comment diff --git a/doc/forum/Using_propellor_for_continers_only/comment_4_9dc985b26c29b9ce21e6c75ec03f6262._comment b/doc/forum/Using_propellor_for_continers_only/comment_4_9dc985b26c29b9ce21e6c75ec03f6262._comment new file mode 100644 index 00000000..72d7ca83 --- /dev/null +++ b/doc/forum/Using_propellor_for_continers_only/comment_4_9dc985b26c29b9ce21e6c75ec03f6262._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2017-05-13T17:42:41Z" + content=""" +The way propellor handles running in a chroot or container is it exports +its binary and support files into the container. This way the +haskell code can run in a container, rather than being limited to +only running shell commands in the container, and without needing ghc in +the container. + +It does use the hardcoded `localdir` for that. +It would certianly be possible to make it use propellor in a different +location, perhaps using `getExecutablePath`. + +Since the git-annex outside the container passes command-line options to +the one running inside the container to tell it what to do, using +`mainProperties` would also not work since that does not look at +command-line options. It would need to use `defaultMain` or +`processCmdLine` and dispatch itself, or something.. +"""]] -- cgit v1.2.3