From a1999115dbb2a9f81a92eea9819c89976daca0d5 Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Tue, 23 May 2023 13:20:48 +0000 Subject: [PATCH] refactor: move container generation to distro modules --- container/container.go | 75 ++++++++++++++- distro/centos/centos.go | 17 ++-- distro/centos/centos_test.go | 22 +++++ distro/debian/debian.go | 26 ++++-- distro/debian/debian_test.go | 17 ++++ distro/oraclelinux/oraclelinux.go | 13 ++- distro/oraclelinux/oraclelinux_test.go | 22 +++++ distro/ubuntu/ubuntu.go | 15 ++- distro/ubuntu/ubuntu_test.go | 22 +++++ go.mod | 3 + kernel.go | 11 +-- kernel/kernel.go | 121 +------------------------ 12 files changed, 212 insertions(+), 152 deletions(-) create mode 100644 distro/centos/centos_test.go create mode 100644 distro/oraclelinux/oraclelinux_test.go create mode 100644 distro/ubuntu/ubuntu_test.go diff --git a/container/container.go b/container/container.go index 1d981c8..32b85d0 100644 --- a/container/container.go +++ b/container/container.go @@ -25,8 +25,12 @@ import ( var Runtime = "docker" +var Registry = "" + var Timeout = time.Hour +var Commands []config.DockerCommand + type Image struct { Name string Distro distro.Distro @@ -140,7 +144,76 @@ func NewFromKernelInfo(ki config.KernelInfo) ( return } -func (c Container) Build(imagePath string) (output string, err error) { +func (c Container) Exist() (yes bool) { + cmd := exec.Command(Runtime, "images", "-q", c.name) + + c.Log.Debug().Msgf("run %v", cmd) + + raw, err := cmd.CombinedOutput() + if err != nil { + c.Log.Error().Err(err).Msg(string(raw)) + return false + } + + yes = string(raw) != "" + + if yes { + c.Log.Debug().Msg("exist") + } else { + c.Log.Debug().Msg("does not exist") + } + + return +} + +func (c Container) Build(image string, envs, runs []string) (err error) { + cdir := config.Dir("containers", c.name) + cfile := filepath.Join(cdir, "Dockerfile") + + cf := "FROM " + if Registry != "" { + cf += Registry + "/" + } + cf += image + "\n" + + for _, c := range Commands { + // TODO check for distro type + cf += "RUN " + c.Command + "\n" + } + + for _, e := range envs { + cf += "ENV " + e + "\n" + } + + for _, c := range runs { + cf += "RUN " + c + "\n" + } + + buf, err := os.ReadFile(cfile) + if err != nil { + err = os.WriteFile(cfile, []byte(cf), os.ModePerm) + if err != nil { + return + } + } + + if string(buf) == cf && c.Exist() { + return + } + + c.Log.Debug().Msg("generate") + + output, err := c.build(cdir) + if err != nil { + c.Log.Error().Err(err).Msg(output) + return + } + + c.Log.Debug().Msg("success") + return +} + +func (c Container) build(imagePath string) (output string, err error) { args := []string{"build"} args = append(args, "-t", c.name, imagePath) diff --git a/distro/centos/centos.go b/distro/centos/centos.go index a98f556..10fe469 100644 --- a/distro/centos/centos.go +++ b/distro/centos/centos.go @@ -47,6 +47,11 @@ func (centos CentOS) Packages() (pkgs []string, err error) { return } + err = c.Build("centos:"+centos.release, centos.envs(), centos.runs()) + if err != nil { + return + } + cmd := "yum search kernel --showduplicates 2>/dev/null " + "| grep '^kernel-[0-9]' " + "| grep -v src " + @@ -63,11 +68,11 @@ func (centos CentOS) Packages() (pkgs []string, err error) { return } -func Envs(km config.Target) (envs []string) { +func (centos CentOS) envs() (envs []string) { return } -func Runs(km config.Target) (commands []string) { +func (centos CentOS) runs() (commands []string) { cmdf := func(f string, s ...interface{}) { commands = append(commands, fmt.Sprintf(f, s...)) } @@ -75,7 +80,7 @@ func Runs(km config.Target) (commands []string) { var repos []string // TODO refactor - switch km.Distro.Release { + switch centos.release { case "6": repofmt := "[6.%d-%s]\\nbaseurl=https://vault.centos.org/6.%d/%s/$basearch/\\ngpgcheck=0" for i := 0; i <= 10; i++ { @@ -109,7 +114,7 @@ func Runs(km config.Target) (commands []string) { repos = append(repos, fmt.Sprintf(repofmt, ver, "appstream", ver, "AppStream")) } default: - log.Fatal().Msgf("no support for %s %s", km.Distro.ID, km.Distro.Release) + log.Fatal().Msgf("no support for centos %s", centos.release) return } @@ -126,14 +131,14 @@ func Runs(km config.Target) (commands []string) { cmdf("yum -y groupinstall 'Development Tools'") - if km.Distro.Release < "8" { + if centos.release < "8" { cmdf("yum -y install deltarpm") } else { cmdf("yum -y install grub2-tools-minimal elfutils-libelf-devel") } var flags string - if km.Distro.Release >= "8" { + if centos.release >= "8" { flags = "--noautoremove" } diff --git a/distro/centos/centos_test.go b/distro/centos/centos_test.go new file mode 100644 index 0000000..5875093 --- /dev/null +++ b/distro/centos/centos_test.go @@ -0,0 +1,22 @@ +package centos + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "code.dumpstack.io/tools/out-of-tree/distro" +) + +func TestCentOS(t *testing.T) { + assert := assert.New(t) + + u := CentOS{release: "7", container: "out_of_tree_centos_7"} + + assert.Equal(u.ID(), distro.CentOS) + assert.Equal(u.Release(), "7") + + assert.True(u.Equal(distro.Distro{Release: "7", ID: distro.CentOS})) + + assert.NotEmpty(u.Packages()) +} diff --git a/distro/debian/debian.go b/distro/debian/debian.go index 32b7a66..22efc03 100644 --- a/distro/debian/debian.go +++ b/distro/debian/debian.go @@ -60,6 +60,16 @@ func (d Debian) Equal(dd distro.Distro) bool { } func (d Debian) Packages() (packages []string, err error) { + c, err := container.New(d.container) + if err != nil { + return + } + + err = c.Build(d.image(), d.envs(), d.runs()) + if err != nil { + return + } + kernels, err := GetKernels() if err != nil { log.Error().Err(err).Msg("get kernels") @@ -178,15 +188,15 @@ func kernelRelease(deb string) (r Release, err error) { return } -func Envs(km config.Target) (envs []string) { +func (d Debian) envs() (envs []string) { envs = append(envs, "DEBIAN_FRONTEND=noninteractive") return } -func ContainerImage(km config.Target) (image string) { +func (d Debian) image() (image string) { image += "debian:" - switch ReleaseFromString(km.Distro.Release) { + switch d.release { case Wheezy: image += "wheezy-20190228" case Jessie: @@ -194,7 +204,7 @@ func ContainerImage(km config.Target) (image string) { case Stretch: image += "stretch-20220622" default: - image += km.Distro.Release + image += d.release.String() } return @@ -231,14 +241,12 @@ func repositories(release Release) (repos []string) { return } -func Runs(km config.Target) (commands []string) { - release := ReleaseFromString(km.Distro.Release) - +func (d Debian) runs() (commands []string) { cmdf := func(f string, s ...interface{}) { commands = append(commands, fmt.Sprintf(f, s...)) } - repos := repositories(release) + repos := repositories(d.release) if len(repos) != 0 { cmdf("rm /etc/apt/sources.list") @@ -260,7 +268,7 @@ func Runs(km config.Target) (commands []string) { "'^(gcc-[0-9].[0-9]|gcc-[0-9])$'", } - if release < 9 { + if d.release < 9 { pkglist = append(pkglist, "module-init-tools") } diff --git a/distro/debian/debian_test.go b/distro/debian/debian_test.go index 6e0f4fe..5a15dca 100644 --- a/distro/debian/debian_test.go +++ b/distro/debian/debian_test.go @@ -2,6 +2,10 @@ package debian import ( "testing" + + "github.com/stretchr/testify/assert" + + "code.dumpstack.io/tools/out-of-tree/distro" ) func TestKernelRelease(t *testing.T) { @@ -38,3 +42,16 @@ func TestKernelRelease(t *testing.T) { } } } + +func TestDebian(t *testing.T) { + assert := assert.New(t) + + u := Debian{release: Wheezy, container: "out_of_tree_debian_7"} + + assert.Equal(u.ID(), distro.Debian) + assert.Equal(u.Release(), "wheezy") + + assert.True(u.Equal(distro.Distro{Release: "wheezy", ID: distro.Debian})) + + assert.NotEmpty(u.Packages()) +} diff --git a/distro/oraclelinux/oraclelinux.go b/distro/oraclelinux/oraclelinux.go index 6e8e38c..6f31e41 100644 --- a/distro/oraclelinux/oraclelinux.go +++ b/distro/oraclelinux/oraclelinux.go @@ -47,6 +47,11 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) { return } + err = c.Build("oraclelinux:"+ol.release, ol.envs(), ol.runs()) + if err != nil { + return + } + cmd := "yum search kernel --showduplicates 2>/dev/null " + "| grep '^kernel-[0-9]\\|^kernel-uek-[0-9]' " + "| grep -v src " + @@ -64,16 +69,16 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) { return } -func Envs(km config.Target) (envs []string) { +func (ol OracleLinux) envs() (envs []string) { return } -func Runs(km config.Target) (commands []string) { +func (ol OracleLinux) runs() (commands []string) { cmdf := func(f string, s ...interface{}) { commands = append(commands, fmt.Sprintf(f, s...)) } - if km.Distro.Release < "6" { + if ol.release < "6" { log.Fatal().Msgf("no support for pre-EL6") } @@ -83,7 +88,7 @@ func Runs(km config.Target) (commands []string) { cmdf("yum -y groupinstall 'Development Tools'") packages := "linux-firmware grubby" - if km.Distro.Release <= "7" { + if ol.release <= "7" { packages += " libdtrace-ctf" } diff --git a/distro/oraclelinux/oraclelinux_test.go b/distro/oraclelinux/oraclelinux_test.go new file mode 100644 index 0000000..7f8ac35 --- /dev/null +++ b/distro/oraclelinux/oraclelinux_test.go @@ -0,0 +1,22 @@ +package oraclelinux + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "code.dumpstack.io/tools/out-of-tree/distro" +) + +func TestOracleLinux(t *testing.T) { + assert := assert.New(t) + + u := OracleLinux{release: "9", container: "out_of_tree_oraclelinux_9"} + + assert.Equal(u.ID(), distro.OracleLinux) + assert.Equal(u.Release(), "9") + + assert.True(u.Equal(distro.Distro{Release: "9", ID: distro.OracleLinux})) + + assert.NotEmpty(u.Packages()) +} diff --git a/distro/ubuntu/ubuntu.go b/distro/ubuntu/ubuntu.go index 9eb62ca..c5dc089 100644 --- a/distro/ubuntu/ubuntu.go +++ b/distro/ubuntu/ubuntu.go @@ -53,6 +53,11 @@ func (u Ubuntu) Packages() (pkgs []string, err error) { return } + err = c.Build("ubuntu:"+u.release, u.envs(), u.runs()) + if err != nil { + return + } + cmd := "apt-cache search " + "--names-only '^linux-image-[0-9\\.\\-]*-generic$' " + "| awk '{ print $1 }'" @@ -69,17 +74,17 @@ func (u Ubuntu) Packages() (pkgs []string, err error) { return } -func Envs(km config.Target) (envs []string) { +func (u Ubuntu) envs() (envs []string) { envs = append(envs, "DEBIAN_FRONTEND=noninteractive") return } -func Runs(km config.Target) (commands []string) { +func (u Ubuntu) runs() (commands []string) { cmdf := func(f string, s ...interface{}) { commands = append(commands, fmt.Sprintf(f, s...)) } - if km.Distro.Release < "14.04" { + if u.release < "14.04" { cmdf("sed -i 's/archive.ubuntu.com/old-releases.ubuntu.com/' " + "/etc/apt/sources.list") } @@ -88,14 +93,14 @@ func Runs(km config.Target) (commands []string) { cmdf("apt-get install -y build-essential libelf-dev") cmdf("apt-get install -y wget git") - if km.Distro.Release == "12.04" { + if u.release == "12.04" { cmdf("apt-get install -y grub") cmdf("cp /bin/true /usr/sbin/grub-probe") cmdf("mkdir -p /boot/grub") cmdf("touch /boot/grub/menu.lst") } - if km.Distro.Release < "14.04" { + if u.release < "14.04" { return } diff --git a/distro/ubuntu/ubuntu_test.go b/distro/ubuntu/ubuntu_test.go new file mode 100644 index 0000000..a0ad020 --- /dev/null +++ b/distro/ubuntu/ubuntu_test.go @@ -0,0 +1,22 @@ +package ubuntu + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "code.dumpstack.io/tools/out-of-tree/distro" +) + +func TestUbuntu(t *testing.T) { + assert := assert.New(t) + + u := Ubuntu{release: "22.04", container: "out_of_tree_ubuntu_22__04"} + + assert.Equal(u.ID(), distro.Ubuntu) + assert.Equal(u.Release(), "22.04") + + assert.True(u.Equal(distro.Distro{Release: "22.04", ID: distro.Ubuntu})) + + assert.NotEmpty(u.Packages()) +} diff --git a/go.mod b/go.mod index ed59d4f..8e093fc 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/rapidloop/skv v0.0.0-20180909015525-9def2caac4cc github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/zerolog v1.29.1 + github.com/stretchr/testify v1.7.0 github.com/zcalusic/sysinfo v0.9.5 golang.org/x/crypto v0.9.0 golang.org/x/time v0.3.0 @@ -43,6 +44,7 @@ require ( github.com/mattn/go-runewidth v0.0.9 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/skeema/knownhosts v1.1.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -50,4 +52,5 @@ require ( golang.org/x/sys v0.8.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.0 // indirect ) diff --git a/kernel.go b/kernel.go index 611f321..112944b 100644 --- a/kernel.go +++ b/kernel.go @@ -75,15 +75,8 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error return } - err = kernel.GenerateBaseDockerImage( - g.Config.Docker.Registry, - g.Config.Docker.Commands, - km, - kernelCmd.Update, - ) - if err != nil { - return - } + container.Registry = g.Config.Docker.Registry + container.Commands = g.Config.Docker.Commands pkgs, err := kernel.MatchPackages(km) // error check skipped on purpose diff --git a/kernel/kernel.go b/kernel/kernel.go index ca7b188..a158b87 100644 --- a/kernel/kernel.go +++ b/kernel/kernel.go @@ -10,7 +10,6 @@ import ( "io/ioutil" "math/rand" "os" - "os/exec" "os/signal" "path/filepath" "regexp" @@ -26,7 +25,6 @@ import ( "code.dumpstack.io/tools/out-of-tree/config" "code.dumpstack.io/tools/out-of-tree/container" "code.dumpstack.io/tools/out-of-tree/distro" - "code.dumpstack.io/tools/out-of-tree/distro/centos" "code.dumpstack.io/tools/out-of-tree/distro/debian" "code.dumpstack.io/tools/out-of-tree/distro/oraclelinux" "code.dumpstack.io/tools/out-of-tree/distro/ubuntu" @@ -71,117 +69,6 @@ func vsyscallAvailable() (available bool, err error) { return } -func GenerateBaseDockerImage(registry string, commands []config.DockerCommand, - sk config.Target, forceUpdate bool) (err error) { - - imagePath := container.ImagePath(sk) - dockerPath := imagePath + "/Dockerfile" - - d := "# BASE\n" - - // TODO move as function to container.go - cmd := exec.Command(container.Runtime, "images", "-q", sk.DockerName()) - log.Debug().Msgf("run %v", cmd) - - rawOutput, err := cmd.CombinedOutput() - if err != nil { - log.Error().Err(err).Msg(string(rawOutput)) - return - } - - if fs.PathExists(dockerPath) && string(rawOutput) != "" { - log.Debug().Msgf("Base image for %s:%s found", - sk.Distro.ID.String(), sk.Distro.Release) - if !forceUpdate { - return - } else { - log.Info().Msgf("Update Containerfile") - } - } - - log.Debug().Msgf("Base image for %s:%s not found, start generating", - sk.Distro.ID.String(), sk.Distro.Release) - os.MkdirAll(imagePath, os.ModePerm) - - d += "FROM " - if registry != "" { - d += registry + "/" - } - - switch sk.Distro.ID { - case distro.Debian: - d += debian.ContainerImage(sk) + "\n" - default: - d += fmt.Sprintf("%s:%s\n", - strings.ToLower(sk.Distro.ID.String()), - sk.Distro.Release) - } - - for _, c := range commands { - d += "RUN " + c.Command + "\n" - } - - // TODO container runs/envs interface - switch sk.Distro.ID { - case distro.Ubuntu: - for _, e := range ubuntu.Envs(sk) { - d += "ENV " + e + "\n" - } - for _, c := range ubuntu.Runs(sk) { - d += "RUN " + c + "\n" - } - case distro.CentOS: - for _, e := range centos.Envs(sk) { - d += "ENV " + e + "\n" - } - for _, c := range centos.Runs(sk) { - d += "RUN " + c + "\n" - } - case distro.OracleLinux: - for _, e := range oraclelinux.Envs(sk) { - d += "ENV " + e + "\n" - } - for _, c := range oraclelinux.Runs(sk) { - d += "RUN " + c + "\n" - } - case distro.Debian: - for _, e := range debian.Envs(sk) { - d += "ENV " + e + "\n" - } - for _, c := range debian.Runs(sk) { - d += "RUN " + c + "\n" - } - default: - err = fmt.Errorf("%s not yet supported", sk.Distro.ID.String()) - return - } - - d += "# END BASE\n\n" - - err = ioutil.WriteFile(dockerPath, []byte(d), 0644) - if err != nil { - return - } - - c, err := container.New(sk.DockerName()) - if err != nil { - return - } - - output, err := c.Build(imagePath) - if err != nil { - log.Error().Err(err).Msgf("Base image for %s:%s generating error", - sk.Distro.ID.String(), sk.Distro.Release) - log.Fatal().Msg(output) - return - } - - log.Debug().Msgf("Base image for %s:%s generating success", - sk.Distro.ID.String(), sk.Distro.Release) - - return -} - func installKernel(sk config.Target, pkgname string, force, headers bool) (err error) { slog := log.With(). Str("distro_type", sk.Distro.ID.String()). @@ -519,6 +406,9 @@ func GenerateKernels(km config.Target, registry string, download, force, headers, shuffle, update bool, shutdown *bool) (err error) { + container.Commands = commands + container.Registry = registry + log.Info().Msgf("Generating for kernel mask %v", km) _, err = GenRootfsImage(container.Image{Name: km.DockerName()}, @@ -527,11 +417,6 @@ func GenerateKernels(km config.Target, registry string, return } - err = GenerateBaseDockerImage(registry, commands, km, update) - if err != nil || *shutdown { - return - } - pkgs, err := MatchPackages(km) if err != nil || *shutdown { return