refactor: move container generation to distro modules

container/container.go

@@ -25,8 +25,12 @@ import (
 
 var Runtime = "docker"
 
+var Registry = ""
+
 var Timeout = time.Hour
 
+var Commands []config.DockerCommand
+
 type Image struct {
 	Name   string
 	Distro distro.Distro
@@ -140,7 +144,76 @@ func NewFromKernelInfo(ki config.KernelInfo) (
 	return
 }
 
-func (c Container) Build(imagePath string) (output string, err error) {
+func (c Container) Exist() (yes bool) {
+	cmd := exec.Command(Runtime, "images", "-q", c.name)
+
+	c.Log.Debug().Msgf("run %v", cmd)
+
+	raw, err := cmd.CombinedOutput()
+	if err != nil {
+		c.Log.Error().Err(err).Msg(string(raw))
+		return false
+	}
+
+	yes = string(raw) != ""
+
+	if yes {
+		c.Log.Debug().Msg("exist")
+	} else {
+		c.Log.Debug().Msg("does not exist")
+	}
+
+	return
+}
+
+func (c Container) Build(image string, envs, runs []string) (err error) {
+	cdir := config.Dir("containers", c.name)
+	cfile := filepath.Join(cdir, "Dockerfile")
+
+	cf := "FROM "
+	if Registry != "" {
+		cf += Registry + "/"
+	}
+	cf += image + "\n"
+
+	for _, c := range Commands {
+		// TODO check for distro type
+		cf += "RUN " + c.Command + "\n"
+	}
+
+	for _, e := range envs {
+		cf += "ENV " + e + "\n"
+	}
+
+	for _, c := range runs {
+		cf += "RUN " + c + "\n"
+	}
+
+	buf, err := os.ReadFile(cfile)
+	if err != nil {
+		err = os.WriteFile(cfile, []byte(cf), os.ModePerm)
+		if err != nil {
+			return
+		}
+	}
+
+	if string(buf) == cf && c.Exist() {
+		return
+	}
+
+	c.Log.Debug().Msg("generate")
+
+	output, err := c.build(cdir)
+	if err != nil {
+		c.Log.Error().Err(err).Msg(output)
+		return
+	}
+
+	c.Log.Debug().Msg("success")
+	return
+}
+
+func (c Container) build(imagePath string) (output string, err error) {
 	args := []string{"build"}
 	args = append(args, "-t", c.name, imagePath)
 

distro/centos/centos.go

@@ -47,6 +47,11 @@ func (centos CentOS) Packages() (pkgs []string, err error) {
 		return
 	}
 
+	err = c.Build("centos:"+centos.release, centos.envs(), centos.runs())
+	if err != nil {
+		return
+	}
+
 	cmd := "yum search kernel --showduplicates 2>/dev/null " +
 		"| grep '^kernel-[0-9]' " +
 		"| grep -v src " +
@@ -63,11 +68,11 @@ func (centos CentOS) Packages() (pkgs []string, err error) {
 	return
 }
 
-func Envs(km config.Target) (envs []string) {
+func (centos CentOS) envs() (envs []string) {
 	return
 }
 
-func Runs(km config.Target) (commands []string) {
+func (centos CentOS) runs() (commands []string) {
 	cmdf := func(f string, s ...interface{}) {
 		commands = append(commands, fmt.Sprintf(f, s...))
 	}
@@ -75,7 +80,7 @@ func Runs(km config.Target) (commands []string) {
 	var repos []string
 
 	// TODO refactor
-	switch km.Distro.Release {
+	switch centos.release {
 	case "6":
 		repofmt := "[6.%d-%s]\\nbaseurl=https://vault.centos.org/6.%d/%s/$basearch/\\ngpgcheck=0"
 		for i := 0; i <= 10; i++ {
@@ -109,7 +114,7 @@ func Runs(km config.Target) (commands []string) {
 			repos = append(repos, fmt.Sprintf(repofmt, ver, "appstream", ver, "AppStream"))
 		}
 	default:
-		log.Fatal().Msgf("no support for %s %s", km.Distro.ID, km.Distro.Release)
+		log.Fatal().Msgf("no support for centos %s", centos.release)
 		return
 	}
 
@@ -126,14 +131,14 @@ func Runs(km config.Target) (commands []string) {
 
 	cmdf("yum -y groupinstall 'Development Tools'")
 
-	if km.Distro.Release < "8" {
+	if centos.release < "8" {
 		cmdf("yum -y install deltarpm")
 	} else {
 		cmdf("yum -y install grub2-tools-minimal elfutils-libelf-devel")
 	}
 
 	var flags string
-	if km.Distro.Release >= "8" {
+	if centos.release >= "8" {
 		flags = "--noautoremove"
 	}
 

distro/centos/centos_test.go

@@ -0,0 +1,22 @@
+package centos
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"code.dumpstack.io/tools/out-of-tree/distro"
+)
+
+func TestCentOS(t *testing.T) {
+	assert := assert.New(t)
+
+	u := CentOS{release: "7", container: "out_of_tree_centos_7"}
+
+	assert.Equal(u.ID(), distro.CentOS)
+	assert.Equal(u.Release(), "7")
+
+	assert.True(u.Equal(distro.Distro{Release: "7", ID: distro.CentOS}))
+
+	assert.NotEmpty(u.Packages())
+}

distro/debian/debian.go

@@ -60,6 +60,16 @@ func (d Debian) Equal(dd distro.Distro) bool {
 }
 
 func (d Debian) Packages() (packages []string, err error) {
+	c, err := container.New(d.container)
+	if err != nil {
+		return
+	}
+
+	err = c.Build(d.image(), d.envs(), d.runs())
+	if err != nil {
+		return
+	}
+
 	kernels, err := GetKernels()
 	if err != nil {
 		log.Error().Err(err).Msg("get kernels")
@@ -178,15 +188,15 @@ func kernelRelease(deb string) (r Release, err error) {
 	return
 }
 
-func Envs(km config.Target) (envs []string) {
+func (d Debian) envs() (envs []string) {
 	envs = append(envs, "DEBIAN_FRONTEND=noninteractive")
 	return
 }
 
-func ContainerImage(km config.Target) (image string) {
+func (d Debian) image() (image string) {
 	image += "debian:"
 
-	switch ReleaseFromString(km.Distro.Release) {
+	switch d.release {
 	case Wheezy:
 		image += "wheezy-20190228"
 	case Jessie:
@@ -194,7 +204,7 @@ func ContainerImage(km config.Target) (image string) {
 	case Stretch:
 		image += "stretch-20220622"
 	default:
-		image += km.Distro.Release
+		image += d.release.String()
 	}
 
 	return
@@ -231,14 +241,12 @@ func repositories(release Release) (repos []string) {
 	return
 }
 
-func Runs(km config.Target) (commands []string) {
-	release := ReleaseFromString(km.Distro.Release)
-
+func (d Debian) runs() (commands []string) {
 	cmdf := func(f string, s ...interface{}) {
 		commands = append(commands, fmt.Sprintf(f, s...))
 	}
 
-	repos := repositories(release)
+	repos := repositories(d.release)
 
 	if len(repos) != 0 {
 		cmdf("rm /etc/apt/sources.list")
@@ -260,7 +268,7 @@ func Runs(km config.Target) (commands []string) {
 		"'^(gcc-[0-9].[0-9]|gcc-[0-9])$'",
 	}
 
-	if release < 9 {
+	if d.release < 9 {
 		pkglist = append(pkglist, "module-init-tools")
 	}
 

distro/debian/debian_test.go

@@ -2,6 +2,10 @@ package debian
 
 import (
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"code.dumpstack.io/tools/out-of-tree/distro"
 )
 
 func TestKernelRelease(t *testing.T) {
@@ -38,3 +42,16 @@ func TestKernelRelease(t *testing.T) {
 		}
 	}
 }
+
+func TestDebian(t *testing.T) {
+	assert := assert.New(t)
+
+	u := Debian{release: Wheezy, container: "out_of_tree_debian_7"}
+
+	assert.Equal(u.ID(), distro.Debian)
+	assert.Equal(u.Release(), "wheezy")
+
+	assert.True(u.Equal(distro.Distro{Release: "wheezy", ID: distro.Debian}))
+
+	assert.NotEmpty(u.Packages())
+}

distro/oraclelinux/oraclelinux.go

@@ -47,6 +47,11 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) {
 		return
 	}
 
+	err = c.Build("oraclelinux:"+ol.release, ol.envs(), ol.runs())
+	if err != nil {
+		return
+	}
+
 	cmd := "yum search kernel --showduplicates 2>/dev/null " +
 		"| grep '^kernel-[0-9]\\|^kernel-uek-[0-9]' " +
 		"| grep -v src " +
@@ -64,16 +69,16 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) {
 	return
 }
 
-func Envs(km config.Target) (envs []string) {
+func (ol OracleLinux) envs() (envs []string) {
 	return
 }
 
-func Runs(km config.Target) (commands []string) {
+func (ol OracleLinux) runs() (commands []string) {
 	cmdf := func(f string, s ...interface{}) {
 		commands = append(commands, fmt.Sprintf(f, s...))
 	}
 
-	if km.Distro.Release < "6" {
+	if ol.release < "6" {
 		log.Fatal().Msgf("no support for pre-EL6")
 	}
 
@@ -83,7 +88,7 @@ func Runs(km config.Target) (commands []string) {
 	cmdf("yum -y groupinstall 'Development Tools'")
 
 	packages := "linux-firmware grubby"
-	if km.Distro.Release <= "7" {
+	if ol.release <= "7" {
 		packages += " libdtrace-ctf"
 	}
 

distro/oraclelinux/oraclelinux_test.go

@@ -0,0 +1,22 @@
+package oraclelinux
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"code.dumpstack.io/tools/out-of-tree/distro"
+)
+
+func TestOracleLinux(t *testing.T) {
+	assert := assert.New(t)
+
+	u := OracleLinux{release: "9", container: "out_of_tree_oraclelinux_9"}
+
+	assert.Equal(u.ID(), distro.OracleLinux)
+	assert.Equal(u.Release(), "9")
+
+	assert.True(u.Equal(distro.Distro{Release: "9", ID: distro.OracleLinux}))
+
+	assert.NotEmpty(u.Packages())
+}

distro/ubuntu/ubuntu.go

@@ -53,6 +53,11 @@ func (u Ubuntu) Packages() (pkgs []string, err error) {
 		return
 	}
 
+	err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
+	if err != nil {
+		return
+	}
+
 	cmd := "apt-cache search " +
 		"--names-only '^linux-image-[0-9\\.\\-]*-generic$' " +
 		"| awk '{ print $1 }'"
@@ -69,17 +74,17 @@ func (u Ubuntu) Packages() (pkgs []string, err error) {
 	return
 }
 
-func Envs(km config.Target) (envs []string) {
+func (u Ubuntu) envs() (envs []string) {
 	envs = append(envs, "DEBIAN_FRONTEND=noninteractive")
 	return
 }
 
-func Runs(km config.Target) (commands []string) {
+func (u Ubuntu) runs() (commands []string) {
 	cmdf := func(f string, s ...interface{}) {
 		commands = append(commands, fmt.Sprintf(f, s...))
 	}
 
-	if km.Distro.Release < "14.04" {
+	if u.release < "14.04" {
 		cmdf("sed -i 's/archive.ubuntu.com/old-releases.ubuntu.com/' " +
 			"/etc/apt/sources.list")
 	}
@@ -88,14 +93,14 @@ func Runs(km config.Target) (commands []string) {
 	cmdf("apt-get install -y build-essential libelf-dev")
 	cmdf("apt-get install -y wget git")
 
-	if km.Distro.Release == "12.04" {
+	if u.release == "12.04" {
 		cmdf("apt-get install -y grub")
 		cmdf("cp /bin/true /usr/sbin/grub-probe")
 		cmdf("mkdir -p /boot/grub")
 		cmdf("touch /boot/grub/menu.lst")
 	}
 
-	if km.Distro.Release < "14.04" {
+	if u.release < "14.04" {
 		return
 	}
 

distro/ubuntu/ubuntu_test.go

@@ -0,0 +1,22 @@
+package ubuntu
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"code.dumpstack.io/tools/out-of-tree/distro"
+)
+
+func TestUbuntu(t *testing.T) {
+	assert := assert.New(t)
+
+	u := Ubuntu{release: "22.04", container: "out_of_tree_ubuntu_22__04"}
+
+	assert.Equal(u.ID(), distro.Ubuntu)
+	assert.Equal(u.Release(), "22.04")
+
+	assert.True(u.Equal(distro.Distro{Release: "22.04", ID: distro.Ubuntu}))
+
+	assert.NotEmpty(u.Packages())
+}

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/rapidloop/skv v0.0.0-20180909015525-9def2caac4cc
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/zerolog v1.29.1
+	github.com/stretchr/testify v1.7.0
 	github.com/zcalusic/sysinfo v0.9.5
 	golang.org/x/crypto v0.9.0
 	golang.org/x/time v0.3.0
@@ -43,6 +44,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
@@ -50,4 +52,5 @@ require (
 	golang.org/x/sys v0.8.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
 )

kernel.go

@@ -75,15 +75,8 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
 		return
 	}
 
-	err = kernel.GenerateBaseDockerImage(
-		g.Config.Docker.Registry,
-		g.Config.Docker.Commands,
-		km,
-		kernelCmd.Update,
-	)
-	if err != nil {
-		return
-	}
+	container.Registry = g.Config.Docker.Registry
+	container.Commands = g.Config.Docker.Commands
 
 	pkgs, err := kernel.MatchPackages(km)
 	// error check skipped on purpose

kernel/kernel.go

@@ -10,7 +10,6 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
-	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"regexp"
@@ -26,7 +25,6 @@ import (
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/container"
 	"code.dumpstack.io/tools/out-of-tree/distro"
-	"code.dumpstack.io/tools/out-of-tree/distro/centos"
 	"code.dumpstack.io/tools/out-of-tree/distro/debian"
 	"code.dumpstack.io/tools/out-of-tree/distro/oraclelinux"
 	"code.dumpstack.io/tools/out-of-tree/distro/ubuntu"
@@ -71,117 +69,6 @@ func vsyscallAvailable() (available bool, err error) {
 	return
 }
 
-func GenerateBaseDockerImage(registry string, commands []config.DockerCommand,
-	sk config.Target, forceUpdate bool) (err error) {
-
-	imagePath := container.ImagePath(sk)
-	dockerPath := imagePath + "/Dockerfile"
-
-	d := "# BASE\n"
-
-	// TODO move as function to container.go
-	cmd := exec.Command(container.Runtime, "images", "-q", sk.DockerName())
-	log.Debug().Msgf("run %v", cmd)
-
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Error().Err(err).Msg(string(rawOutput))
-		return
-	}
-
-	if fs.PathExists(dockerPath) && string(rawOutput) != "" {
-		log.Debug().Msgf("Base image for %s:%s found",
-			sk.Distro.ID.String(), sk.Distro.Release)
-		if !forceUpdate {
-			return
-		} else {
-			log.Info().Msgf("Update Containerfile")
-		}
-	}
-
-	log.Debug().Msgf("Base image for %s:%s not found, start generating",
-		sk.Distro.ID.String(), sk.Distro.Release)
-	os.MkdirAll(imagePath, os.ModePerm)
-
-	d += "FROM "
-	if registry != "" {
-		d += registry + "/"
-	}
-
-	switch sk.Distro.ID {
-	case distro.Debian:
-		d += debian.ContainerImage(sk) + "\n"
-	default:
-		d += fmt.Sprintf("%s:%s\n",
-			strings.ToLower(sk.Distro.ID.String()),
-			sk.Distro.Release)
-	}
-
-	for _, c := range commands {
-		d += "RUN " + c.Command + "\n"
-	}
-
-	// TODO container runs/envs interface
-	switch sk.Distro.ID {
-	case distro.Ubuntu:
-		for _, e := range ubuntu.Envs(sk) {
-			d += "ENV " + e + "\n"
-		}
-		for _, c := range ubuntu.Runs(sk) {
-			d += "RUN " + c + "\n"
-		}
-	case distro.CentOS:
-		for _, e := range centos.Envs(sk) {
-			d += "ENV " + e + "\n"
-		}
-		for _, c := range centos.Runs(sk) {
-			d += "RUN " + c + "\n"
-		}
-	case distro.OracleLinux:
-		for _, e := range oraclelinux.Envs(sk) {
-			d += "ENV " + e + "\n"
-		}
-		for _, c := range oraclelinux.Runs(sk) {
-			d += "RUN " + c + "\n"
-		}
-	case distro.Debian:
-		for _, e := range debian.Envs(sk) {
-			d += "ENV " + e + "\n"
-		}
-		for _, c := range debian.Runs(sk) {
-			d += "RUN " + c + "\n"
-		}
-	default:
-		err = fmt.Errorf("%s not yet supported", sk.Distro.ID.String())
-		return
-	}
-
-	d += "# END BASE\n\n"
-
-	err = ioutil.WriteFile(dockerPath, []byte(d), 0644)
-	if err != nil {
-		return
-	}
-
-	c, err := container.New(sk.DockerName())
-	if err != nil {
-		return
-	}
-
-	output, err := c.Build(imagePath)
-	if err != nil {
-		log.Error().Err(err).Msgf("Base image for %s:%s generating error",
-			sk.Distro.ID.String(), sk.Distro.Release)
-		log.Fatal().Msg(output)
-		return
-	}
-
-	log.Debug().Msgf("Base image for %s:%s generating success",
-		sk.Distro.ID.String(), sk.Distro.Release)
-
-	return
-}
-
 func installKernel(sk config.Target, pkgname string, force, headers bool) (err error) {
 	slog := log.With().
 		Str("distro_type", sk.Distro.ID.String()).
@@ -519,6 +406,9 @@ func GenerateKernels(km config.Target, registry string,
 	download, force, headers, shuffle, update bool,
 	shutdown *bool) (err error) {
 
+	container.Commands = commands
+	container.Registry = registry
+
 	log.Info().Msgf("Generating for kernel mask %v", km)
 
 	_, err = GenRootfsImage(container.Image{Name: km.DockerName()},
@@ -527,11 +417,6 @@ func GenerateKernels(km config.Target, registry string,
 		return
 	}
 
-	err = GenerateBaseDockerImage(registry, commands, km, update)
-	if err != nil || *shutdown {
-		return
-	}
-
 	pkgs, err := MatchPackages(km)
 	if err != nil || *shutdown {
 		return