From d5ccc10d91fd2773e963c4f948f3cfc7f024e1ba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 30 Mar 2016 13:07:11 -0400 Subject: problem --- ...lem_with_spin_after_new_dependencies_added.mdwn | 46 ++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 doc/todo/problem_with_spin_after_new_dependencies_added.mdwn diff --git a/doc/todo/problem_with_spin_after_new_dependencies_added.mdwn b/doc/todo/problem_with_spin_after_new_dependencies_added.mdwn new file mode 100644 index 00000000..3a2c97af --- /dev/null +++ b/doc/todo/problem_with_spin_after_new_dependencies_added.mdwn @@ -0,0 +1,46 @@ +Using --spin against a remote host after new cabal deps are added fails. + +
+Sending git update to clam.kitenet.net ... done
+Pull from central git repository ... done
+git branch origin/joeyconfig gpg signature verified; merging
+Already up-to-date.
+Warning: The package list for 'hackage.haskell.org' is 77.9 days old.
+Run 'cabal update' to get the latest list of available packages.
+Resolving dependencies...
+Configuring propellor-3.0.0...
+cabal: At least the following dependencies are missing:
+concurrent-output -any
+propellor: failed to make dist/setup-config
+Shared connection to clam.kitenet.net closed.
+propellor: remote propellor failed
+- exit 1
+
+ +This is a blocker for merging the typed-os-requirements branch. + +Problem is, the remote propellor runs Propellor.Bootstrap.build to build +itself after updating the git tree. But that does not install any missing +cabal deps. + +It can be made to, but there are three problems: + +1. For it to use depsCommand to install missing deps, it needs to know + the target OS of the host it's running on. That would need to be + extracted from other info. +2. Propellor.Bootstrap.build is also run when local propellor builds + itself, and if cabal failed there, it's not running as root and so can't + install deps. And probably shouldn't try to anyhow. +3. Even if Propellor.Bootstrap.build is fixed to install deps, + this would still require an upgrade to get that fix before new deps can + be added. This presents difficulties in merging the + typed-os-requirements branch. + +Instead of fixing this in Propellor.Bootstrap.build, could it be fixed +in the --spin code? That could run the depsCommand, but that's too +expensive to do every time. Only need to do it if the remote +propellor's build of itself fails. How to tell when that happened, +vs when propellor built ok, ran, and exited nonzero due to a failing property? + +(Note that the cron job runs the depsCommand if cabal configure fails, +so deps do get installed that way, only --spin is a problem.) -- cgit v1.2.3 From d051f2b366cbdfdbf879176094cb2c3c4ad67391 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 30 Mar 2016 13:08:58 -0400 Subject: close --- doc/todo/missing_dependencies.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/missing_dependencies.mdwn b/doc/todo/missing_dependencies.mdwn index 55490a86..2b2ac0f4 100644 --- a/doc/todo/missing_dependencies.mdwn +++ b/doc/todo/missing_dependencies.mdwn @@ -37,3 +37,5 @@ After upgrading to 2.4.0, I get this error: propellor: remote propellor failed As in https://propellor.branchable.com/todo/issue_after_upgrading_shared_library/, manually running "make clean" on the server fixed the issue + +> Ok, this is the same as [[problem_with_spin_after_new_dependencies_added]]. Closing this issue as I'm dealing with it in the other one. [[done]] --[[Joey]] -- cgit v1.2.3 From bc607807f9aa7dffea4e6e3267b4874acd64747f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 30 Mar 2016 15:18:39 -0400 Subject: When new dependencies are added to propellor or the propellor config, try harder to get them installed. In particular, this makes propellor --spin work when the remote host needs to get dependencies installed in order to build the updated config. Fixes http://propellor.branchable.com/todo/problem_with_spin_after_new_dependencies_added/ (cherry picked from commit 0f410f8acdb9e0b84ae364e80e5ee63adcb2ee50) --- debian/changelog | 13 +++++++++++-- src/Propellor/Bootstrap.hs | 33 ++++++++++++++++++++++++--------- src/Propellor/CmdLine.hs | 30 +++++++++++++++--------------- src/wrapper.hs | 2 +- 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/debian/changelog b/debian/changelog index b85c836f..88aaf917 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +propellor (2.17.2) unstable; urgency=medium + + * When new dependencies are added to propellor or the propellor config, + try harder to get them installed. In particular, this makes + propellor --spin work when the remote host needs to get dependencies + installed in order to build the updated config. + * Apt.update: Also run dpkg --configure -a here as apt for some reason + won't even update if dpkg was interrupted. + + -- Joey Hess Mon, 28 Mar 2016 11:06:34 -0400 + propellor (2.17.1) unstable; urgency=medium * Avoid generating excessively long paths to the unix socket file @@ -6,8 +17,6 @@ propellor (2.17.1) unstable; urgency=medium * Uwsgi: add ".ini" extension to app config files. Files without extensions were ignored by uwsgi. Thanks, FĂ©lix Sipma. - * Apt.update: Also run dpkg --configure -a here as apt for some reason - won't even update if dpkg was interrupted. -- Joey Hess Mon, 28 Mar 2016 11:06:34 -0400 diff --git a/src/Propellor/Bootstrap.hs b/src/Propellor/Bootstrap.hs index 69eee66c..b81a2302 100644 --- a/src/Propellor/Bootstrap.hs +++ b/src/Propellor/Bootstrap.hs @@ -6,6 +6,7 @@ module Propellor.Bootstrap ( ) where import Propellor.Base +import Propellor.Types.Info import System.Posix.Files import Data.List @@ -128,22 +129,27 @@ installGitCommand msys = case msys of , "DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends --no-upgrade -y install git" ] -buildPropellor :: IO () -buildPropellor = unlessM (actionMessage "Propellor build" build) $ +buildPropellor :: Maybe Host -> IO () +buildPropellor mh = unlessM (actionMessage "Propellor build" (build msys)) $ errorMessage "Propellor build failed!" + where + msys = case fmap (getInfo . hostInfo) mh of + Just (InfoVal sys) -> Just sys + _ -> Nothing -- Build propellor using cabal, and symlink propellor to where cabal -- leaves the built binary. -- -- For speed, only runs cabal configure when it's not been run before. -- If the build fails cabal may need to have configure re-run. -build :: IO Bool -build = catchBoolIO $ do - make "dist/setup-config" ["propellor.cabal"] $ - cabal ["configure"] - unlessM (cabal ["build", "propellor-config"]) $ do - void $ cabal ["configure"] - unlessM (cabal ["build"]) $ +-- +-- If the cabal configure fails, and a System is provided, installs +-- dependencies and retries. +build :: Maybe System -> IO Bool +build msys = catchBoolIO $ do + make "dist/setup-config" ["propellor.cabal"] cabal_configure + unlessM cabal_build $ + unlessM (cabal_configure <&&> cabal_build) $ error "cabal build failed" -- For safety against eg power loss in the middle of the build, -- make a copy of the binary, and move it into place atomically. @@ -163,6 +169,15 @@ build = catchBoolIO $ do cabalbuiltbin = "dist/build/propellor-config/propellor-config" safetycopy = cabalbuiltbin ++ ".built" tmpfor f = f ++ ".propellortmp" + cabal_configure = ifM (cabal ["configure"]) + ( return True + , case msys of + Nothing -> return False + Just sys -> + boolSystem "sh" [Param "-c", Param (depsCommand (Just sys))] + <&&> cabal ["configure"] + ) + cabal_build = cabal ["build", "propellor-config"] make :: FilePath -> [FilePath] -> IO Bool -> IO () make dest srcs builder = do diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index ee057d05..8fd2bf18 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -114,20 +114,20 @@ defaultMain hostlist = withConcurrentOutput $ do go _ (DockerInit hn) = Docker.init hn go _ (GitPush fin fout) = gitPushHelper fin fout go cr (Relay h) = forceConsole >> - updateFirst cr (Update (Just h)) (update (Just h)) + updateFirst Nothing cr (Update (Just h)) (update (Just h)) go _ (Update Nothing) = forceConsole >> fetchFirst (onlyprocess (update Nothing)) go _ (Update (Just h)) = update (Just h) go _ Merge = mergeSpin - go cr cmdline@(Spin hs mrelay) = buildFirst cr cmdline $ do + go cr cmdline@(Spin hs mrelay) = buildFirst Nothing cr cmdline $ do unless (isJust mrelay) commitSpin forM_ hs $ \hn -> withhost hn $ spin mrelay hn go cr cmdline@(Run hn) = ifM ((==) 0 <$> getRealUserID) - ( updateFirst cr cmdline $ runhost hn + ( updateFirst (findHost hostlist hn) cr cmdline $ runhost hn , fetchFirst $ go cr (Spin [hn] Nothing) ) go cr cmdline@(SimpleRun hn) = forceConsole >> - fetchFirst (buildFirst cr cmdline (runhost hn)) + fetchFirst (buildFirst (findHost hostlist hn) cr cmdline (runhost hn)) -- When continuing after a rebuild, don't want to rebuild again. go _ (Continue cmdline) = go NoRebuild cmdline @@ -149,17 +149,17 @@ unknownhost h hosts = errorMessage $ unlines -- Builds propellor (when allowed) and if it looks like a new binary, -- re-execs it to continue. -- Otherwise, runs the IO action to continue. -buildFirst :: CanRebuild -> CmdLine -> IO () -> IO () -buildFirst CanRebuild cmdline next = do +buildFirst :: Maybe Host -> CanRebuild -> CmdLine -> IO () -> IO () +buildFirst h CanRebuild cmdline next = do oldtime <- getmtime - buildPropellor + buildPropellor h newtime <- getmtime if newtime == oldtime then next else continueAfterBuild cmdline where getmtime = catchMaybeIO $ getModificationTime "propellor" -buildFirst NoRebuild _ next = next +buildFirst _ NoRebuild _ next = next continueAfterBuild :: CmdLine -> IO a continueAfterBuild cmdline = go =<< boolSystem "./propellor" @@ -176,23 +176,23 @@ fetchFirst next = do void fetchOrigin next -updateFirst :: CanRebuild -> CmdLine -> IO () -> IO () -updateFirst canrebuild cmdline next = ifM hasOrigin - ( updateFirst' canrebuild cmdline next +updateFirst :: Maybe Host -> CanRebuild -> CmdLine -> IO () -> IO () +updateFirst h canrebuild cmdline next = ifM hasOrigin + ( updateFirst' h canrebuild cmdline next , next ) -- If changes can be fetched from origin, Builds propellor (when allowed) -- and re-execs the updated propellor binary to continue. -- Otherwise, runs the IO action to continue. -updateFirst' :: CanRebuild -> CmdLine -> IO () -> IO () -updateFirst' CanRebuild cmdline next = ifM fetchOrigin +updateFirst' :: Maybe Host -> CanRebuild -> CmdLine -> IO () -> IO () +updateFirst' h CanRebuild cmdline next = ifM fetchOrigin ( do - buildPropellor + buildPropellor h continueAfterBuild cmdline , next ) -updateFirst' NoRebuild _ next = next +updateFirst' _ NoRebuild _ next = next -- Gets the fully qualified domain name, given a string that might be -- a short name to look up in the DNS. diff --git a/src/wrapper.hs b/src/wrapper.hs index a204b60c..289b12b5 100644 --- a/src/wrapper.hs +++ b/src/wrapper.hs @@ -99,7 +99,7 @@ wrapper args propellordir propellorbin = do warnoutofdate propellordir True buildruncfg = do changeWorkingDirectory propellordir - buildPropellor + buildPropellor Nothing putStrLn "" putStrLn "" chain -- cgit v1.2.3