fix: avoid rebuild of prebuilt containers

.github/workflows/e2e.yml

@@ -139,7 +139,7 @@ jobs:
         echo 'Type=oneshot' >> test.service
         echo 'WorkingDirectory=/root/test' >> test.service
         echo 'TimeoutStopSec=1' >> test.service
-        echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-container-cache autogen --threads=8 --max=64 --shuffle' >> test.service
+        echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-prebuilt-containers autogen --threads=8 --max=64 --shuffle' >> test.service
         echo 'ExecStart=/usr/local/bin/out-of-tree pew --qemu-timeout=10m --threads=4 --include-internal-errors' >> test.service
 
         scp test.service root@$IP:/etc/systemd/system/test.service

cmd/kernel.go

@@ -11,34 +11,31 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cavaliergopher/grab/v3"
 	"github.com/naoina/toml"
 	"github.com/remeh/sizedwaitgroup"
 	"github.com/rs/zerolog/log"
 
 	"code.dumpstack.io/tools/out-of-tree/artifact"
-	"code.dumpstack.io/tools/out-of-tree/cache"
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
 	"code.dumpstack.io/tools/out-of-tree/container"
 	"code.dumpstack.io/tools/out-of-tree/distro"
-	"code.dumpstack.io/tools/out-of-tree/fs"
 	"code.dumpstack.io/tools/out-of-tree/kernel"
 )
 
 type KernelCmd struct {
-	NoDownload     bool `help:"do not download qemu image while kernel generation"`
+	NoDownload         bool `help:"do not download qemu image while kernel generation"`
-	UseHost        bool `help:"also use host kernels"`
+	UseHost            bool `help:"also use host kernels"`
-	Force          bool `help:"force reinstall kernel"`
+	Force              bool `help:"force reinstall kernel"`
-	NoHeaders      bool `help:"do not install kernel headers"`
+	NoHeaders          bool `help:"do not install kernel headers"`
-	Shuffle        bool `help:"randomize kernels installation order"`
+	Shuffle            bool `help:"randomize kernels installation order"`
-	Retries        int  `help:"amount of tries for each kernel" default:"2"`
+	Retries            int  `help:"amount of tries for each kernel" default:"2"`
-	Threads        int  `help:"threads for parallel installation" default:"1"`
+	Threads            int  `help:"threads for parallel installation" default:"1"`
-	Update         bool `help:"update container"`
+	Update             bool `help:"update container"`
-	ContainerCache bool `help:"try prebuilt container images first" default:"true" negatable:""`
+	PrebuiltContainers bool `help:"try prebuilt container images first" default:"true" negatable:""`
-	Max            int  `help:"maximum kernels to download" default:"100500"`
+	Max                int  `help:"maximum kernels to download" default:"100500"`
-	NoPrune        bool `help:"do not remove dangling or unused images from local storage after build"`
+	NoPrune            bool `help:"do not remove dangling or unused images from local storage after build"`
-	NoCfgRegen     bool `help:"do not update kernels.toml"`
+	NoCfgRegen         bool `help:"do not update kernels.toml"`
 
 	ContainerTimeout time.Duration `help:"container timeout"`
 
@@ -168,33 +165,6 @@ func (cmd *KernelCmd) GenKernel(km artifact.Target, pkg string) {
 	}
 }
 
-func (cmd *KernelCmd) fetchContainerCache(c container.Container) {
-	if !cmd.ContainerCache {
-		return
-	}
-	if c.Exist() {
-		return
-	}
-
-	tmp, err := fs.TempDir()
-	if err != nil {
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	resp, err := grab.Get(tmp, cache.ContainerURL(c.Name()))
-	if err != nil {
-		return
-	}
-
-	defer os.Remove(resp.Filename)
-
-	err = container.Load(resp.Filename, c.Name())
-	if err == nil {
-		log.Info().Msgf("use prebuilt container %s", c.Name())
-	}
-}
-
 func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
 	defer func() {
 		if err != nil {
@@ -211,6 +181,8 @@ func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
 		container.Prune = false
 	}
 
+	container.UsePrebuilt = cmd.PrebuiltContainers
+
 	cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
 	if err != nil {
 		log.Debug().Err(err).Msg("read kernels config")
@@ -230,13 +202,6 @@ func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
 		return
 	}
 
-	c, err := container.New(km.Distro)
-	if err != nil || cmd.shutdown {
-		return
-	}
-
-	cmd.fetchContainerCache(c)
-
 	pkgs, err := kernel.MatchPackages(km)
 	if err != nil || cmd.shutdown {
 		return
@@ -312,6 +277,8 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
 		container.Prune = false
 	}
 
+	container.UsePrebuilt = kernelCmd.PrebuiltContainers
+
 	distroType, err := distro.NewID(cmd.Distro)
 	if err != nil {
 		return
@@ -330,13 +297,6 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
 	container.Registry = g.Config.Docker.Registry
 	container.Commands = g.Config.Docker.Commands
 
-	c, err := container.New(km.Distro)
-	if err != nil {
-		return
-	}
-
-	kernelCmd.fetchContainerCache(c)
-
 	pkgs, err := kernel.MatchPackages(km)
 	// error check skipped on purpose
 

container/container.go

@@ -16,11 +16,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cavaliergopher/grab/v3"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 
+	"code.dumpstack.io/tools/out-of-tree/cache"
 	"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
 	"code.dumpstack.io/tools/out-of-tree/distro"
+	"code.dumpstack.io/tools/out-of-tree/fs"
 )
 
 var Runtime = "docker"
@@ -33,6 +36,8 @@ var Commands []distro.Command
 
 var UseCache = true
 
+var UsePrebuilt = true
+
 var Prune = true
 
 type Image struct {
@@ -240,7 +245,38 @@ func (c Container) Exist() (yes bool) {
 	return
 }
 
+func (c Container) loadPrebuilt() (err error) {
+	if c.Exist() && UseCache {
+		return
+	}
+
+	tmp, err := fs.TempDir()
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	log.Info().Msgf("download prebuilt container %s", c.Name())
+	resp, err := grab.Get(tmp, cache.ContainerURL(c.Name()))
+	if err != nil {
+		return
+	}
+
+	defer os.Remove(resp.Filename)
+
+	err = Load(resp.Filename, c.Name())
+	if err == nil {
+		log.Info().Msgf("use prebuilt container %s", c.Name())
+	}
+
+	return
+}
+
 func (c Container) Build(image string, envs, runs []string) (err error) {
+	if c.Exist() && UseCache {
+		return
+	}
+
 	cdir := dotfiles.Dir("containers", c.name)
 	cfile := filepath.Join(cdir, "Dockerfile")
 
@@ -286,10 +322,17 @@ func (c Container) Build(image string, envs, runs []string) (err error) {
 		c.Log.Info().Msg("build")
 	}
 
-	output, err := c.build(cdir)
+	if UsePrebuilt {
-	if err != nil {
+		err = c.loadPrebuilt()
-		c.Log.Error().Err(err).Msg(output)
+	}
-		return
+
+	if err != nil || !UsePrebuilt {
+		var output string
+		output, err = c.build(cdir)
+		if err != nil {
+			c.Log.Error().Err(err).Msg(output)
+			return
+		}
 	}
 
 	c.Log.Info().Msg("success")

distro/centos/centos.go

@@ -37,12 +37,9 @@ func (centos CentOS) Packages() (pkgs []string, err error) {
 		return
 	}
 
-	if !c.Exist() {
+	err = c.Build("centos:"+centos.release, centos.envs(), centos.runs())
-		err = c.Build("centos:"+centos.release,
+	if err != nil {
-			centos.envs(), centos.runs())
+		return
-		if err != nil {
-			return
-		}
 	}
 
 	cmd := "yum search kernel --showduplicates 2>/dev/null " +

distro/debian/debian.go

@@ -54,11 +54,9 @@ func (d Debian) Packages() (packages []string, err error) {
 		return
 	}
 
-	if !c.Exist() {
+	err = c.Build(d.image(), d.envs(), d.runs())
-		err = c.Build(d.image(), d.envs(), d.runs())
+	if err != nil {
-		if err != nil {
+		return
-			return
-		}
 	}
 
 	kernels, err := GetKernels()

distro/opensuse/opensuse.go

@@ -70,11 +70,9 @@ func (suse OpenSUSE) Packages() (pkgs []string, err error) {
 		name = "opensuse/leap:" + suse.release
 	}
 
-	if !c.Exist() {
+	err = c.Build(name, suse.envs(), suse.runs())
-		err = c.Build(name, suse.envs(), suse.runs())
+	if err != nil {
-		if err != nil {
+		return
-			return
-		}
 	}
 
 	cmd := "zypper search -s --match-exact kernel-default | grep x86_64 " +

distro/oraclelinux/oraclelinux.go

@@ -38,12 +38,9 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) {
 		return
 	}
 
-	if !c.Exist() {
+	err = c.Build("oraclelinux:"+ol.release, ol.envs(), ol.runs())
-		err = c.Build("oraclelinux:"+ol.release,
+	if err != nil {
-			ol.envs(), ol.runs())
+		return
-		if err != nil {
-			return
-		}
 	}
 
 	if ol.release == "8" {

distro/ubuntu/ubuntu.go

@@ -43,11 +43,9 @@ func (u Ubuntu) Packages() (pkgs []string, err error) {
 		return
 	}
 
-	if !c.Exist() {
+	err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
-		err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
+	if err != nil {
-		if err != nil {
+		return
-			return
-		}
 	}
 
 	cmd := "apt-cache search " +