1
0

refactor: move container generation to distro modules

This commit is contained in:
dump_stack() 2023-05-23 13:20:48 +00:00
parent ff7bed76f2
commit a1999115db
Signed by: dump_stack
GPG Key ID: BE44DA8C062D87DC
12 changed files with 212 additions and 152 deletions

View File

@ -25,8 +25,12 @@ import (
var Runtime = "docker" var Runtime = "docker"
var Registry = ""
var Timeout = time.Hour var Timeout = time.Hour
var Commands []config.DockerCommand
type Image struct { type Image struct {
Name string Name string
Distro distro.Distro Distro distro.Distro
@ -140,7 +144,76 @@ func NewFromKernelInfo(ki config.KernelInfo) (
return return
} }
func (c Container) Build(imagePath string) (output string, err error) { func (c Container) Exist() (yes bool) {
cmd := exec.Command(Runtime, "images", "-q", c.name)
c.Log.Debug().Msgf("run %v", cmd)
raw, err := cmd.CombinedOutput()
if err != nil {
c.Log.Error().Err(err).Msg(string(raw))
return false
}
yes = string(raw) != ""
if yes {
c.Log.Debug().Msg("exist")
} else {
c.Log.Debug().Msg("does not exist")
}
return
}
func (c Container) Build(image string, envs, runs []string) (err error) {
cdir := config.Dir("containers", c.name)
cfile := filepath.Join(cdir, "Dockerfile")
cf := "FROM "
if Registry != "" {
cf += Registry + "/"
}
cf += image + "\n"
for _, c := range Commands {
// TODO check for distro type
cf += "RUN " + c.Command + "\n"
}
for _, e := range envs {
cf += "ENV " + e + "\n"
}
for _, c := range runs {
cf += "RUN " + c + "\n"
}
buf, err := os.ReadFile(cfile)
if err != nil {
err = os.WriteFile(cfile, []byte(cf), os.ModePerm)
if err != nil {
return
}
}
if string(buf) == cf && c.Exist() {
return
}
c.Log.Debug().Msg("generate")
output, err := c.build(cdir)
if err != nil {
c.Log.Error().Err(err).Msg(output)
return
}
c.Log.Debug().Msg("success")
return
}
func (c Container) build(imagePath string) (output string, err error) {
args := []string{"build"} args := []string{"build"}
args = append(args, "-t", c.name, imagePath) args = append(args, "-t", c.name, imagePath)

View File

@ -47,6 +47,11 @@ func (centos CentOS) Packages() (pkgs []string, err error) {
return return
} }
err = c.Build("centos:"+centos.release, centos.envs(), centos.runs())
if err != nil {
return
}
cmd := "yum search kernel --showduplicates 2>/dev/null " + cmd := "yum search kernel --showduplicates 2>/dev/null " +
"| grep '^kernel-[0-9]' " + "| grep '^kernel-[0-9]' " +
"| grep -v src " + "| grep -v src " +
@ -63,11 +68,11 @@ func (centos CentOS) Packages() (pkgs []string, err error) {
return return
} }
func Envs(km config.Target) (envs []string) { func (centos CentOS) envs() (envs []string) {
return return
} }
func Runs(km config.Target) (commands []string) { func (centos CentOS) runs() (commands []string) {
cmdf := func(f string, s ...interface{}) { cmdf := func(f string, s ...interface{}) {
commands = append(commands, fmt.Sprintf(f, s...)) commands = append(commands, fmt.Sprintf(f, s...))
} }
@ -75,7 +80,7 @@ func Runs(km config.Target) (commands []string) {
var repos []string var repos []string
// TODO refactor // TODO refactor
switch km.Distro.Release { switch centos.release {
case "6": case "6":
repofmt := "[6.%d-%s]\\nbaseurl=https://vault.centos.org/6.%d/%s/$basearch/\\ngpgcheck=0" repofmt := "[6.%d-%s]\\nbaseurl=https://vault.centos.org/6.%d/%s/$basearch/\\ngpgcheck=0"
for i := 0; i <= 10; i++ { for i := 0; i <= 10; i++ {
@ -109,7 +114,7 @@ func Runs(km config.Target) (commands []string) {
repos = append(repos, fmt.Sprintf(repofmt, ver, "appstream", ver, "AppStream")) repos = append(repos, fmt.Sprintf(repofmt, ver, "appstream", ver, "AppStream"))
} }
default: default:
log.Fatal().Msgf("no support for %s %s", km.Distro.ID, km.Distro.Release) log.Fatal().Msgf("no support for centos %s", centos.release)
return return
} }
@ -126,14 +131,14 @@ func Runs(km config.Target) (commands []string) {
cmdf("yum -y groupinstall 'Development Tools'") cmdf("yum -y groupinstall 'Development Tools'")
if km.Distro.Release < "8" { if centos.release < "8" {
cmdf("yum -y install deltarpm") cmdf("yum -y install deltarpm")
} else { } else {
cmdf("yum -y install grub2-tools-minimal elfutils-libelf-devel") cmdf("yum -y install grub2-tools-minimal elfutils-libelf-devel")
} }
var flags string var flags string
if km.Distro.Release >= "8" { if centos.release >= "8" {
flags = "--noautoremove" flags = "--noautoremove"
} }

View File

@ -0,0 +1,22 @@
package centos
import (
"testing"
"github.com/stretchr/testify/assert"
"code.dumpstack.io/tools/out-of-tree/distro"
)
func TestCentOS(t *testing.T) {
assert := assert.New(t)
u := CentOS{release: "7", container: "out_of_tree_centos_7"}
assert.Equal(u.ID(), distro.CentOS)
assert.Equal(u.Release(), "7")
assert.True(u.Equal(distro.Distro{Release: "7", ID: distro.CentOS}))
assert.NotEmpty(u.Packages())
}

View File

@ -60,6 +60,16 @@ func (d Debian) Equal(dd distro.Distro) bool {
} }
func (d Debian) Packages() (packages []string, err error) { func (d Debian) Packages() (packages []string, err error) {
c, err := container.New(d.container)
if err != nil {
return
}
err = c.Build(d.image(), d.envs(), d.runs())
if err != nil {
return
}
kernels, err := GetKernels() kernels, err := GetKernels()
if err != nil { if err != nil {
log.Error().Err(err).Msg("get kernels") log.Error().Err(err).Msg("get kernels")
@ -178,15 +188,15 @@ func kernelRelease(deb string) (r Release, err error) {
return return
} }
func Envs(km config.Target) (envs []string) { func (d Debian) envs() (envs []string) {
envs = append(envs, "DEBIAN_FRONTEND=noninteractive") envs = append(envs, "DEBIAN_FRONTEND=noninteractive")
return return
} }
func ContainerImage(km config.Target) (image string) { func (d Debian) image() (image string) {
image += "debian:" image += "debian:"
switch ReleaseFromString(km.Distro.Release) { switch d.release {
case Wheezy: case Wheezy:
image += "wheezy-20190228" image += "wheezy-20190228"
case Jessie: case Jessie:
@ -194,7 +204,7 @@ func ContainerImage(km config.Target) (image string) {
case Stretch: case Stretch:
image += "stretch-20220622" image += "stretch-20220622"
default: default:
image += km.Distro.Release image += d.release.String()
} }
return return
@ -231,14 +241,12 @@ func repositories(release Release) (repos []string) {
return return
} }
func Runs(km config.Target) (commands []string) { func (d Debian) runs() (commands []string) {
release := ReleaseFromString(km.Distro.Release)
cmdf := func(f string, s ...interface{}) { cmdf := func(f string, s ...interface{}) {
commands = append(commands, fmt.Sprintf(f, s...)) commands = append(commands, fmt.Sprintf(f, s...))
} }
repos := repositories(release) repos := repositories(d.release)
if len(repos) != 0 { if len(repos) != 0 {
cmdf("rm /etc/apt/sources.list") cmdf("rm /etc/apt/sources.list")
@ -260,7 +268,7 @@ func Runs(km config.Target) (commands []string) {
"'^(gcc-[0-9].[0-9]|gcc-[0-9])$'", "'^(gcc-[0-9].[0-9]|gcc-[0-9])$'",
} }
if release < 9 { if d.release < 9 {
pkglist = append(pkglist, "module-init-tools") pkglist = append(pkglist, "module-init-tools")
} }

View File

@ -2,6 +2,10 @@ package debian
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert"
"code.dumpstack.io/tools/out-of-tree/distro"
) )
func TestKernelRelease(t *testing.T) { func TestKernelRelease(t *testing.T) {
@ -38,3 +42,16 @@ func TestKernelRelease(t *testing.T) {
} }
} }
} }
func TestDebian(t *testing.T) {
assert := assert.New(t)
u := Debian{release: Wheezy, container: "out_of_tree_debian_7"}
assert.Equal(u.ID(), distro.Debian)
assert.Equal(u.Release(), "wheezy")
assert.True(u.Equal(distro.Distro{Release: "wheezy", ID: distro.Debian}))
assert.NotEmpty(u.Packages())
}

View File

@ -47,6 +47,11 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) {
return return
} }
err = c.Build("oraclelinux:"+ol.release, ol.envs(), ol.runs())
if err != nil {
return
}
cmd := "yum search kernel --showduplicates 2>/dev/null " + cmd := "yum search kernel --showduplicates 2>/dev/null " +
"| grep '^kernel-[0-9]\\|^kernel-uek-[0-9]' " + "| grep '^kernel-[0-9]\\|^kernel-uek-[0-9]' " +
"| grep -v src " + "| grep -v src " +
@ -64,16 +69,16 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) {
return return
} }
func Envs(km config.Target) (envs []string) { func (ol OracleLinux) envs() (envs []string) {
return return
} }
func Runs(km config.Target) (commands []string) { func (ol OracleLinux) runs() (commands []string) {
cmdf := func(f string, s ...interface{}) { cmdf := func(f string, s ...interface{}) {
commands = append(commands, fmt.Sprintf(f, s...)) commands = append(commands, fmt.Sprintf(f, s...))
} }
if km.Distro.Release < "6" { if ol.release < "6" {
log.Fatal().Msgf("no support for pre-EL6") log.Fatal().Msgf("no support for pre-EL6")
} }
@ -83,7 +88,7 @@ func Runs(km config.Target) (commands []string) {
cmdf("yum -y groupinstall 'Development Tools'") cmdf("yum -y groupinstall 'Development Tools'")
packages := "linux-firmware grubby" packages := "linux-firmware grubby"
if km.Distro.Release <= "7" { if ol.release <= "7" {
packages += " libdtrace-ctf" packages += " libdtrace-ctf"
} }

View File

@ -0,0 +1,22 @@
package oraclelinux
import (
"testing"
"github.com/stretchr/testify/assert"
"code.dumpstack.io/tools/out-of-tree/distro"
)
func TestOracleLinux(t *testing.T) {
assert := assert.New(t)
u := OracleLinux{release: "9", container: "out_of_tree_oraclelinux_9"}
assert.Equal(u.ID(), distro.OracleLinux)
assert.Equal(u.Release(), "9")
assert.True(u.Equal(distro.Distro{Release: "9", ID: distro.OracleLinux}))
assert.NotEmpty(u.Packages())
}

View File

@ -53,6 +53,11 @@ func (u Ubuntu) Packages() (pkgs []string, err error) {
return return
} }
err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
if err != nil {
return
}
cmd := "apt-cache search " + cmd := "apt-cache search " +
"--names-only '^linux-image-[0-9\\.\\-]*-generic$' " + "--names-only '^linux-image-[0-9\\.\\-]*-generic$' " +
"| awk '{ print $1 }'" "| awk '{ print $1 }'"
@ -69,17 +74,17 @@ func (u Ubuntu) Packages() (pkgs []string, err error) {
return return
} }
func Envs(km config.Target) (envs []string) { func (u Ubuntu) envs() (envs []string) {
envs = append(envs, "DEBIAN_FRONTEND=noninteractive") envs = append(envs, "DEBIAN_FRONTEND=noninteractive")
return return
} }
func Runs(km config.Target) (commands []string) { func (u Ubuntu) runs() (commands []string) {
cmdf := func(f string, s ...interface{}) { cmdf := func(f string, s ...interface{}) {
commands = append(commands, fmt.Sprintf(f, s...)) commands = append(commands, fmt.Sprintf(f, s...))
} }
if km.Distro.Release < "14.04" { if u.release < "14.04" {
cmdf("sed -i 's/archive.ubuntu.com/old-releases.ubuntu.com/' " + cmdf("sed -i 's/archive.ubuntu.com/old-releases.ubuntu.com/' " +
"/etc/apt/sources.list") "/etc/apt/sources.list")
} }
@ -88,14 +93,14 @@ func Runs(km config.Target) (commands []string) {
cmdf("apt-get install -y build-essential libelf-dev") cmdf("apt-get install -y build-essential libelf-dev")
cmdf("apt-get install -y wget git") cmdf("apt-get install -y wget git")
if km.Distro.Release == "12.04" { if u.release == "12.04" {
cmdf("apt-get install -y grub") cmdf("apt-get install -y grub")
cmdf("cp /bin/true /usr/sbin/grub-probe") cmdf("cp /bin/true /usr/sbin/grub-probe")
cmdf("mkdir -p /boot/grub") cmdf("mkdir -p /boot/grub")
cmdf("touch /boot/grub/menu.lst") cmdf("touch /boot/grub/menu.lst")
} }
if km.Distro.Release < "14.04" { if u.release < "14.04" {
return return
} }

View File

@ -0,0 +1,22 @@
package ubuntu
import (
"testing"
"github.com/stretchr/testify/assert"
"code.dumpstack.io/tools/out-of-tree/distro"
)
func TestUbuntu(t *testing.T) {
assert := assert.New(t)
u := Ubuntu{release: "22.04", container: "out_of_tree_ubuntu_22__04"}
assert.Equal(u.ID(), distro.Ubuntu)
assert.Equal(u.Release(), "22.04")
assert.True(u.Equal(distro.Distro{Release: "22.04", ID: distro.Ubuntu}))
assert.NotEmpty(u.Packages())
}

3
go.mod
View File

@ -18,6 +18,7 @@ require (
github.com/rapidloop/skv v0.0.0-20180909015525-9def2caac4cc github.com/rapidloop/skv v0.0.0-20180909015525-9def2caac4cc
github.com/remeh/sizedwaitgroup v1.0.0 github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/zerolog v1.29.1 github.com/rs/zerolog v1.29.1
github.com/stretchr/testify v1.7.0
github.com/zcalusic/sysinfo v0.9.5 github.com/zcalusic/sysinfo v0.9.5
golang.org/x/crypto v0.9.0 golang.org/x/crypto v0.9.0
golang.org/x/time v0.3.0 golang.org/x/time v0.3.0
@ -43,6 +44,7 @@ require (
github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sergi/go-diff v1.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect
github.com/skeema/knownhosts v1.1.0 // indirect github.com/skeema/knownhosts v1.1.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect
@ -50,4 +52,5 @@ require (
golang.org/x/sys v0.8.0 // indirect golang.org/x/sys v0.8.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.0 // indirect
) )

View File

@ -75,15 +75,8 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
return return
} }
err = kernel.GenerateBaseDockerImage( container.Registry = g.Config.Docker.Registry
g.Config.Docker.Registry, container.Commands = g.Config.Docker.Commands
g.Config.Docker.Commands,
km,
kernelCmd.Update,
)
if err != nil {
return
}
pkgs, err := kernel.MatchPackages(km) pkgs, err := kernel.MatchPackages(km)
// error check skipped on purpose // error check skipped on purpose

View File

@ -10,7 +10,6 @@ import (
"io/ioutil" "io/ioutil"
"math/rand" "math/rand"
"os" "os"
"os/exec"
"os/signal" "os/signal"
"path/filepath" "path/filepath"
"regexp" "regexp"
@ -26,7 +25,6 @@ import (
"code.dumpstack.io/tools/out-of-tree/config" "code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/container" "code.dumpstack.io/tools/out-of-tree/container"
"code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/distro"
"code.dumpstack.io/tools/out-of-tree/distro/centos"
"code.dumpstack.io/tools/out-of-tree/distro/debian" "code.dumpstack.io/tools/out-of-tree/distro/debian"
"code.dumpstack.io/tools/out-of-tree/distro/oraclelinux" "code.dumpstack.io/tools/out-of-tree/distro/oraclelinux"
"code.dumpstack.io/tools/out-of-tree/distro/ubuntu" "code.dumpstack.io/tools/out-of-tree/distro/ubuntu"
@ -71,117 +69,6 @@ func vsyscallAvailable() (available bool, err error) {
return return
} }
func GenerateBaseDockerImage(registry string, commands []config.DockerCommand,
sk config.Target, forceUpdate bool) (err error) {
imagePath := container.ImagePath(sk)
dockerPath := imagePath + "/Dockerfile"
d := "# BASE\n"
// TODO move as function to container.go
cmd := exec.Command(container.Runtime, "images", "-q", sk.DockerName())
log.Debug().Msgf("run %v", cmd)
rawOutput, err := cmd.CombinedOutput()
if err != nil {
log.Error().Err(err).Msg(string(rawOutput))
return
}
if fs.PathExists(dockerPath) && string(rawOutput) != "" {
log.Debug().Msgf("Base image for %s:%s found",
sk.Distro.ID.String(), sk.Distro.Release)
if !forceUpdate {
return
} else {
log.Info().Msgf("Update Containerfile")
}
}
log.Debug().Msgf("Base image for %s:%s not found, start generating",
sk.Distro.ID.String(), sk.Distro.Release)
os.MkdirAll(imagePath, os.ModePerm)
d += "FROM "
if registry != "" {
d += registry + "/"
}
switch sk.Distro.ID {
case distro.Debian:
d += debian.ContainerImage(sk) + "\n"
default:
d += fmt.Sprintf("%s:%s\n",
strings.ToLower(sk.Distro.ID.String()),
sk.Distro.Release)
}
for _, c := range commands {
d += "RUN " + c.Command + "\n"
}
// TODO container runs/envs interface
switch sk.Distro.ID {
case distro.Ubuntu:
for _, e := range ubuntu.Envs(sk) {
d += "ENV " + e + "\n"
}
for _, c := range ubuntu.Runs(sk) {
d += "RUN " + c + "\n"
}
case distro.CentOS:
for _, e := range centos.Envs(sk) {
d += "ENV " + e + "\n"
}
for _, c := range centos.Runs(sk) {
d += "RUN " + c + "\n"
}
case distro.OracleLinux:
for _, e := range oraclelinux.Envs(sk) {
d += "ENV " + e + "\n"
}
for _, c := range oraclelinux.Runs(sk) {
d += "RUN " + c + "\n"
}
case distro.Debian:
for _, e := range debian.Envs(sk) {
d += "ENV " + e + "\n"
}
for _, c := range debian.Runs(sk) {
d += "RUN " + c + "\n"
}
default:
err = fmt.Errorf("%s not yet supported", sk.Distro.ID.String())
return
}
d += "# END BASE\n\n"
err = ioutil.WriteFile(dockerPath, []byte(d), 0644)
if err != nil {
return
}
c, err := container.New(sk.DockerName())
if err != nil {
return
}
output, err := c.Build(imagePath)
if err != nil {
log.Error().Err(err).Msgf("Base image for %s:%s generating error",
sk.Distro.ID.String(), sk.Distro.Release)
log.Fatal().Msg(output)
return
}
log.Debug().Msgf("Base image for %s:%s generating success",
sk.Distro.ID.String(), sk.Distro.Release)
return
}
func installKernel(sk config.Target, pkgname string, force, headers bool) (err error) { func installKernel(sk config.Target, pkgname string, force, headers bool) (err error) {
slog := log.With(). slog := log.With().
Str("distro_type", sk.Distro.ID.String()). Str("distro_type", sk.Distro.ID.String()).
@ -519,6 +406,9 @@ func GenerateKernels(km config.Target, registry string,
download, force, headers, shuffle, update bool, download, force, headers, shuffle, update bool,
shutdown *bool) (err error) { shutdown *bool) (err error) {
container.Commands = commands
container.Registry = registry
log.Info().Msgf("Generating for kernel mask %v", km) log.Info().Msgf("Generating for kernel mask %v", km)
_, err = GenRootfsImage(container.Image{Name: km.DockerName()}, _, err = GenRootfsImage(container.Image{Name: km.DockerName()},
@ -527,11 +417,6 @@ func GenerateKernels(km config.Target, registry string,
return return
} }
err = GenerateBaseDockerImage(registry, commands, km, update)
if err != nil || *shutdown {
return
}
pkgs, err := MatchPackages(km) pkgs, err := MatchPackages(km)
if err != nil || *shutdown { if err != nil || *shutdown {
return return