.github/workflows/e2e.yml

@@ -139,7 +139,7 @@ jobs:
         echo 'Type=oneshot' >> test.service
         echo 'WorkingDirectory=/root/test' >> test.service
         echo 'TimeoutStopSec=1' >> test.service
-        echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-prebuilt-containers autogen --threads=8 --max=64 --shuffle' >> test.service
+        echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-container-cache autogen --threads=8 --max=64 --shuffle' >> test.service
         echo 'ExecStart=/usr/local/bin/out-of-tree pew --qemu-timeout=10m --threads=4 --include-internal-errors' >> test.service
 
         scp test.service root@$IP:/etc/systemd/system/test.service

cmd/kernel.go

@@ -11,31 +11,34 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cavaliergopher/grab/v3"
 	"github.com/naoina/toml"
 	"github.com/remeh/sizedwaitgroup"
 	"github.com/rs/zerolog/log"
 
 	"code.dumpstack.io/tools/out-of-tree/artifact"
+	"code.dumpstack.io/tools/out-of-tree/cache"
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
 	"code.dumpstack.io/tools/out-of-tree/container"
 	"code.dumpstack.io/tools/out-of-tree/distro"
+	"code.dumpstack.io/tools/out-of-tree/fs"
 	"code.dumpstack.io/tools/out-of-tree/kernel"
 )
 
 type KernelCmd struct {
-	NoDownload         bool `help:"do not download qemu image while kernel generation"`
-	UseHost            bool `help:"also use host kernels"`
-	Force              bool `help:"force reinstall kernel"`
-	NoHeaders          bool `help:"do not install kernel headers"`
-	Shuffle            bool `help:"randomize kernels installation order"`
-	Retries            int  `help:"amount of tries for each kernel" default:"2"`
-	Threads            int  `help:"threads for parallel installation" default:"1"`
-	Update             bool `help:"update container"`
-	PrebuiltContainers bool `help:"try prebuilt container images first" default:"true" negatable:""`
-	Max                int  `help:"maximum kernels to download" default:"100500"`
-	NoPrune            bool `help:"do not remove dangling or unused images from local storage after build"`
-	NoCfgRegen         bool `help:"do not update kernels.toml"`
+	NoDownload     bool `help:"do not download qemu image while kernel generation"`
+	UseHost        bool `help:"also use host kernels"`
+	Force          bool `help:"force reinstall kernel"`
+	NoHeaders      bool `help:"do not install kernel headers"`
+	Shuffle        bool `help:"randomize kernels installation order"`
+	Retries        int  `help:"amount of tries for each kernel" default:"2"`
+	Threads        int  `help:"threads for parallel installation" default:"1"`
+	Update         bool `help:"update container"`
+	ContainerCache bool `help:"try prebuilt container images first" default:"true" negatable:""`
+	Max            int  `help:"maximum kernels to download" default:"100500"`
+	NoPrune        bool `help:"do not remove dangling or unused images from local storage after build"`
+	NoCfgRegen     bool `help:"do not update kernels.toml"`
 
 	ContainerTimeout time.Duration `help:"container timeout"`
 
@@ -165,6 +168,33 @@ func (cmd *KernelCmd) GenKernel(km artifact.Target, pkg string) {
 	}
 }
 
+func (cmd *KernelCmd) fetchContainerCache(c container.Container) {
+	if !cmd.ContainerCache {
+		return
+	}
+	if c.Exist() {
+		return
+	}
+
+	tmp, err := fs.TempDir()
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	resp, err := grab.Get(tmp, cache.ContainerURL(c.Name()))
+	if err != nil {
+		return
+	}
+
+	defer os.Remove(resp.Filename)
+
+	err = container.Load(resp.Filename, c.Name())
+	if err == nil {
+		log.Info().Msgf("use prebuilt container %s", c.Name())
+	}
+}
+
 func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
 	defer func() {
 		if err != nil {
@@ -181,8 +211,6 @@ func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
 		container.Prune = false
 	}
 
-	container.UsePrebuilt = cmd.PrebuiltContainers
-
 	cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
 	if err != nil {
 		log.Debug().Err(err).Msg("read kernels config")
@@ -202,6 +230,13 @@ func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
 		return
 	}
 
+	c, err := container.New(km.Distro)
+	if err != nil || cmd.shutdown {
+		return
+	}
+
+	cmd.fetchContainerCache(c)
+
 	pkgs, err := kernel.MatchPackages(km)
 	if err != nil || cmd.shutdown {
 		return
@@ -277,8 +312,6 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
 		container.Prune = false
 	}
 
-	container.UsePrebuilt = kernelCmd.PrebuiltContainers
-
 	distroType, err := distro.NewID(cmd.Distro)
 	if err != nil {
 		return
@@ -297,6 +330,13 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
 	container.Registry = g.Config.Docker.Registry
 	container.Commands = g.Config.Docker.Commands
 
+	c, err := container.New(km.Distro)
+	if err != nil {
+		return
+	}
+
+	kernelCmd.fetchContainerCache(c)
+
 	pkgs, err := kernel.MatchPackages(km)
 	// error check skipped on purpose
 

container/container.go

@@ -16,14 +16,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cavaliergopher/grab/v3"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 
-	"code.dumpstack.io/tools/out-of-tree/cache"
 	"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
 	"code.dumpstack.io/tools/out-of-tree/distro"
-	"code.dumpstack.io/tools/out-of-tree/fs"
 )
 
 var Runtime = "docker"
@@ -36,8 +33,6 @@ var Commands []distro.Command
 
 var UseCache = true
 
-var UsePrebuilt = true
-
 var Prune = true
 
 type Image struct {
@@ -245,38 +240,7 @@ func (c Container) Exist() (yes bool) {
 	return
 }
 
-func (c Container) loadPrebuilt() (err error) {
-	if c.Exist() && UseCache {
-		return
-	}
-
-	tmp, err := fs.TempDir()
-	if err != nil {
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	log.Info().Msgf("download prebuilt container %s", c.Name())
-	resp, err := grab.Get(tmp, cache.ContainerURL(c.Name()))
-	if err != nil {
-		return
-	}
-
-	defer os.Remove(resp.Filename)
-
-	err = Load(resp.Filename, c.Name())
-	if err == nil {
-		log.Info().Msgf("use prebuilt container %s", c.Name())
-	}
-
-	return
-}
-
 func (c Container) Build(image string, envs, runs []string) (err error) {
-	if c.Exist() && UseCache {
-		return
-	}
-
 	cdir := dotfiles.Dir("containers", c.name)
 	cfile := filepath.Join(cdir, "Dockerfile")
 
@@ -322,17 +286,10 @@ func (c Container) Build(image string, envs, runs []string) (err error) {
 		c.Log.Info().Msg("build")
 	}
 
-	if UsePrebuilt {
-		err = c.loadPrebuilt()
-	}
-
-	if err != nil || !UsePrebuilt {
-		var output string
-		output, err = c.build(cdir)
-		if err != nil {
-			c.Log.Error().Err(err).Msg(output)
-			return
-		}
+	output, err := c.build(cdir)
+	if err != nil {
+		c.Log.Error().Err(err).Msg(output)
+		return
 	}
 
 	c.Log.Info().Msg("success")

distro/centos/centos.go

@@ -37,9 +37,12 @@ func (centos CentOS) Packages() (pkgs []string, err error) {
 		return
 	}
 
-	err = c.Build("centos:"+centos.release, centos.envs(), centos.runs())
-	if err != nil {
-		return
+	if !c.Exist() {
+		err = c.Build("centos:"+centos.release,
+			centos.envs(), centos.runs())
+		if err != nil {
+			return
+		}
 	}
 
 	cmd := "yum search kernel --showduplicates 2>/dev/null " +

distro/debian/debian.go

@@ -54,9 +54,11 @@ func (d Debian) Packages() (packages []string, err error) {
 		return
 	}
 
-	err = c.Build(d.image(), d.envs(), d.runs())
-	if err != nil {
-		return
+	if !c.Exist() {
+		err = c.Build(d.image(), d.envs(), d.runs())
+		if err != nil {
+			return
+		}
 	}
 
 	kernels, err := GetKernels()

distro/opensuse/opensuse.go

@@ -70,9 +70,11 @@ func (suse OpenSUSE) Packages() (pkgs []string, err error) {
 		name = "opensuse/leap:" + suse.release
 	}
 
-	err = c.Build(name, suse.envs(), suse.runs())
-	if err != nil {
-		return
+	if !c.Exist() {
+		err = c.Build(name, suse.envs(), suse.runs())
+		if err != nil {
+			return
+		}
 	}
 
 	cmd := "zypper search -s --match-exact kernel-default | grep x86_64 " +

distro/oraclelinux/oraclelinux.go

@@ -38,9 +38,12 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) {
 		return
 	}
 
-	err = c.Build("oraclelinux:"+ol.release, ol.envs(), ol.runs())
-	if err != nil {
-		return
+	if !c.Exist() {
+		err = c.Build("oraclelinux:"+ol.release,
+			ol.envs(), ol.runs())
+		if err != nil {
+			return
+		}
 	}
 
 	if ol.release == "8" {

distro/ubuntu/ubuntu.go

@@ -43,9 +43,11 @@ func (u Ubuntu) Packages() (pkgs []string, err error) {
 		return
 	}
 
-	err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
-	if err != nil {
-		return
+	if !c.Exist() {
+		err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
+		if err != nil {
+			return
+		}
 	}
 
 	cmd := "apt-cache search " +