From bcf8de336f5309d903de18215c34293745b81f35 Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Thu, 18 May 2023 16:07:24 +0000 Subject: [PATCH] feat!: introduce new distribution structure BREAKING CHANGE: distro definition in the configuration files has switched from [[supported_kernels]] distro_type = "Ubuntu" distro_release = "16.04" ... to [[supported_kernels]] distro = { id = "Ubuntu", release = "16.04" } ... --- cache/cache.go | 6 +- config/config.go | 87 +++-------------------- config/config_test.go | 7 +- config/out-of-tree.go | 6 +- container/container.go | 16 ++--- db.go | 10 +-- distro.go | 4 +- distro/centos/centos.go | 8 +-- distro/debian/debian.go | 13 ++-- distro/debian/debian_test.go | 5 +- distro/distro.go | 68 ++++++++++++++++++ distro/oraclelinux/oraclelinux.go | 6 +- distro/ubuntu/ubuntu.go | 4 +- examples/kernel-exploit/.out-of-tree.toml | 15 ++-- examples/kernel-module/.out-of-tree.toml | 12 ++-- examples/preload/.out-of-tree.toml | 3 +- examples/script/.out-of-tree.toml | 3 +- gen.go | 10 ++- kernel.go | 26 ++++--- kernel/kernel.go | 62 ++++++++-------- kernel/kernel_linux.go | 16 +++-- log.go | 18 ++--- pew.go | 27 +++---- preload.go | 6 +- 24 files changed, 219 insertions(+), 219 deletions(-) create mode 100644 distro/distro.go diff --git a/cache/cache.go b/cache/cache.go index db0fb21..2f384e5 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -15,7 +15,7 @@ import ( "github.com/cavaliergopher/grab/v3" "github.com/rs/zerolog/log" - "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/fs" ) @@ -95,8 +95,8 @@ func DownloadDebianCache(cachePath string) (err error) { return os.Rename(resp.Filename, cachePath) } -func PackageURL(dt config.DistroType, orig string) (found bool, fileurl string) { - if dt != config.Debian { +func PackageURL(dt distro.ID, orig string) (found bool, fileurl string) { + if dt != distro.Debian { return } diff --git a/config/config.go b/config/config.go index d589134..0007206 100644 --- a/config/config.go +++ b/config/config.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "code.dumpstack.io/tools/out-of-tree/distro" + "github.com/naoina/toml" ) @@ -26,9 +28,9 @@ type kernel struct { // KernelMask defines the kernel type KernelMask struct { - DistroType DistroType - DistroRelease string // 18.04/7/9 - ReleaseMask string + Distro distro.Distro + + ReleaseMask string // Overrides ReleaseMask Kernel kernel @@ -36,8 +38,8 @@ type KernelMask struct { // DockerName is returns stable name for docker container func (km KernelMask) DockerName() string { - distro := strings.ToLower(km.DistroType.String()) - release := strings.Replace(km.DistroRelease, ".", "__", -1) + distro := strings.ToLower(km.Distro.ID.String()) + release := strings.Replace(km.Distro.Release, ".", "__", -1) return fmt.Sprintf("out_of_tree_%s_%s", distro, release) } @@ -168,13 +170,13 @@ type Artifact struct { func (ka Artifact) checkSupport(ki KernelInfo, km KernelMask) ( supported bool, err error) { - if ki.DistroType != km.DistroType { + if ki.Distro.ID != km.Distro.ID { supported = false return } // DistroRelease is optional - if km.DistroRelease != "" && ki.DistroRelease != km.DistroRelease { + if km.Distro.Release != "" && ki.Distro.Release != km.Distro.Release { supported = false return } @@ -195,74 +197,6 @@ func (ka Artifact) Supported(ki KernelInfo) (supported bool, err error) { return } -// DistroType is enum with all supported distros -type DistroType int - -const ( - // Ubuntu https://ubuntu.com/ - Ubuntu DistroType = iota - // CentOS https://www.centos.org/ - CentOS - // Debian https://www.debian.org/ - Debian - // OracleLinux https://www.oracle.com/linux/ - OracleLinux -) - -// DistroTypeStrings is the string version of enum DistroType -var DistroTypeStrings = [...]string{ - "Ubuntu", - "CentOS", - "Debian", - "OracleLinux", -} - -// NewDistroType is create new Distro object -func NewDistroType(dType string) (dt DistroType, err error) { - err = dt.UnmarshalTOML([]byte(dType)) - return -} - -func (dt DistroType) String() string { - return DistroTypeStrings[dt] -} - -// UnmarshalTOML is for support github.com/naoina/toml -func (dt *DistroType) UnmarshalTOML(data []byte) (err error) { - sDistro := strings.Trim(string(data), `"`) - if strings.EqualFold(sDistro, "Ubuntu") { - *dt = Ubuntu - } else if strings.EqualFold(sDistro, "CentOS") { - *dt = CentOS - } else if strings.EqualFold(sDistro, "Debian") { - *dt = Debian - } else if strings.EqualFold(sDistro, "OracleLinux") { - *dt = OracleLinux - } else { - err = fmt.Errorf("Distro %s is unsupported", sDistro) - } - return -} - -// MarshalTOML is for support github.com/naoina/toml -func (dt DistroType) MarshalTOML() (data []byte, err error) { - s := "" - switch dt { - case Ubuntu: - s = "Ubuntu" - case CentOS: - s = "CentOS" - case Debian: - s = "Debian" - case OracleLinux: - s = "OracleLinux" - default: - err = fmt.Errorf("Cannot marshal %d", dt) - } - data = []byte(`"` + s + `"`) - return -} - // ByRootFS is sorting by .RootFS lexicographically type ByRootFS []KernelInfo @@ -272,8 +206,7 @@ func (a ByRootFS) Less(i, j int) bool { return a[i].RootFS < a[j].RootFS } // KernelInfo defines kernels.toml entries type KernelInfo struct { - DistroType DistroType - DistroRelease string // 18.04/7.4.1708/9.1 + Distro distro.Distro // Must be *exactly* same as in `uname -r` KernelVersion string diff --git a/config/config_test.go b/config/config_test.go index c8cff5f..ba8ce77 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -7,6 +7,8 @@ package config import ( "testing" + "code.dumpstack.io/tools/out-of-tree/distro" + "github.com/naoina/toml" ) @@ -16,7 +18,10 @@ func TestMarshalUnmarshal(t *testing.T) { Type: KernelModule, } artifactCfg.SupportedKernels = append(artifactCfg.SupportedKernels, - KernelMask{Ubuntu, "18.04", ".*", kernel{}}) + KernelMask{distro.Distro{ + ID: distro.Ubuntu, + Release: "18.04", + }, ".*", kernel{}}) buf, err := toml.Marshal(&artifactCfg) if err != nil { t.Fatal(err) diff --git a/config/out-of-tree.go b/config/out-of-tree.go index 8fb855f..408712f 100644 --- a/config/out-of-tree.go +++ b/config/out-of-tree.go @@ -9,14 +9,16 @@ import ( "os" "time" + "code.dumpstack.io/tools/out-of-tree/distro" + "github.com/alecthomas/kong" "github.com/mitchellh/go-homedir" "github.com/naoina/toml" ) type DockerCommand struct { - DistroType DistroType - Command string + Distro distro.Distro + Command string } type OutOfTree struct { diff --git a/container/container.go b/container/container.go index 82f934a..a53de26 100644 --- a/container/container.go +++ b/container/container.go @@ -21,14 +21,14 @@ import ( "github.com/rs/zerolog/log" "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/distro" ) var Runtime = "docker" type Image struct { - Name string - DistroType config.DistroType - DistroRelease string // 18.04/7/9 + Name string + Distro distro.Distro } func Images() (diis []Image, err error) { @@ -51,14 +51,14 @@ func Images() (diis []Image, err error) { s := strings.Replace(containerName, "__", ".", -1) values := strings.Split(s, "_") - distro, ver := values[3], values[4] + distroName, ver := values[3], values[4] dii := Image{ - Name: containerName, - DistroRelease: ver, + Name: containerName, } - dii.DistroType, err = config.NewDistroType(distro) + dii.Distro.Release = ver + dii.Distro.ID, err = distro.NewID(distroName) if err != nil { return } @@ -69,7 +69,7 @@ func Images() (diis []Image, err error) { } func ImagePath(sk config.KernelMask) string { - return config.Dir("containers", sk.DistroType.String(), sk.DistroRelease) + return config.Dir("containers", sk.Distro.ID.String(), sk.Distro.Release) } type Volumes struct { diff --git a/db.go b/db.go index 71e9db9..0e14280 100644 --- a/db.go +++ b/db.go @@ -140,7 +140,7 @@ func addToLog(db *sql.DB, q *qemu.System, ka config.Artifact, _, err = stmt.Exec( ka.Name, ka.Type, tag, - ki.DistroType, ki.DistroRelease, ki.KernelRelease, + ki.Distro.ID, ki.Distro.Release, ki.KernelRelease, res.Build.Output, res.Build.Ok, res.Run.Output, res.Run.Ok, res.Test.Output, res.Test.Ok, @@ -175,7 +175,7 @@ func getAllLogs(db *sql.DB, tag string, num int) (les []logEntry, err error) { le := logEntry{} err = rows.Scan(&le.ID, &le.Timestamp, &le.Name, &le.Type, &le.Tag, - &le.DistroType, &le.DistroRelease, &le.KernelRelease, + &le.Distro.ID, &le.Distro.Release, &le.KernelRelease, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.KernelPanic, &le.KilledByTimeout, ) @@ -214,7 +214,7 @@ func getAllArtifactLogs(db *sql.DB, tag string, num int, ka config.Artifact) ( le := logEntry{} err = rows.Scan(&le.ID, &le.Timestamp, &le.Name, &le.Type, &le.Tag, - &le.DistroType, &le.DistroRelease, &le.KernelRelease, + &le.Distro.ID, &le.Distro.Release, &le.KernelRelease, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.KernelPanic, &le.KilledByTimeout, ) @@ -245,7 +245,7 @@ func getLogByID(db *sql.DB, id int) (le logEntry, err error) { err = stmt.QueryRow(id).Scan(&le.ID, &le.Timestamp, &le.Name, &le.Type, &le.Tag, - &le.DistroType, &le.DistroRelease, &le.KernelRelease, + &le.Distro.ID, &le.Distro.Release, &le.KernelRelease, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.Build.Output, &le.Run.Output, &le.Test.Output, &le.Stdout, &le.Stderr, @@ -263,7 +263,7 @@ func getLastLog(db *sql.DB) (le logEntry, err error) { "kernel_panic, timeout_kill "+ "FROM log").Scan(&le.ID, &le.Timestamp, &le.Name, &le.Type, &le.Tag, - &le.DistroType, &le.DistroRelease, &le.KernelRelease, + &le.Distro.ID, &le.Distro.Release, &le.KernelRelease, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.Build.Output, &le.Run.Output, &le.Test.Output, &le.Stdout, &le.Stderr, diff --git a/distro.go b/distro.go index f2bcec2..b340b21 100644 --- a/distro.go +++ b/distro.go @@ -14,7 +14,7 @@ import ( "github.com/rs/zerolog/log" "code.dumpstack.io/tools/out-of-tree/cache" - "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/distro/debian" "code.dumpstack.io/tools/out-of-tree/distro/debian/snapshot" "code.dumpstack.io/tools/out-of-tree/fs" @@ -92,7 +92,7 @@ func (cmd *DebianFetchCmd) fetch(pkg snapshot.Package) { if !cmd.IgnoreMirror { flog.Debug().Msg("check mirror") - found, _ := cache.PackageURL(config.Debian, pkg.Deb.URL) + found, _ := cache.PackageURL(distro.Debian, pkg.Deb.URL) if found { flog.Debug().Msg("found on the mirror") return diff --git a/distro/centos/centos.go b/distro/centos/centos.go index cca937e..5ba0a02 100644 --- a/distro/centos/centos.go +++ b/distro/centos/centos.go @@ -20,7 +20,7 @@ func Runs(km config.KernelMask) (commands []string) { var repos []string // TODO refactor - switch km.DistroRelease { + switch km.Distro.Release { case "6": repofmt := "[6.%d-%s]\\nbaseurl=https://vault.centos.org/6.%d/%s/$basearch/\\ngpgcheck=0" for i := 0; i <= 10; i++ { @@ -54,7 +54,7 @@ func Runs(km config.KernelMask) (commands []string) { repos = append(repos, fmt.Sprintf(repofmt, ver, "appstream", ver, "AppStream")) } default: - log.Fatal().Msgf("no support for %s %s", km.DistroType, km.DistroRelease) + log.Fatal().Msgf("no support for %s %s", km.Distro.ID, km.Distro.Release) return } @@ -71,14 +71,14 @@ func Runs(km config.KernelMask) (commands []string) { cmdf("yum -y groupinstall 'Development Tools'") - if km.DistroRelease < "8" { + if km.Distro.Release < "8" { cmdf("yum -y install deltarpm") } else { cmdf("yum -y install grub2-tools-minimal elfutils-libelf-devel") } var flags string - if km.DistroRelease >= "8" { + if km.Distro.Release >= "8" { flags = "--noautoremove" } diff --git a/distro/debian/debian.go b/distro/debian/debian.go index 773ef62..09c6f62 100644 --- a/distro/debian/debian.go +++ b/distro/debian/debian.go @@ -118,7 +118,7 @@ func Match(km config.KernelMask) (pkgs []string, err error) { return } - release := releaseFromString(km.DistroRelease) + release := releaseFromString(km.Distro.Release) r := regexp.MustCompile(km.ReleaseMask) @@ -151,7 +151,7 @@ func Envs(km config.KernelMask) (envs []string) { func ContainerImage(km config.KernelMask) (image string) { image += "debian:" - switch releaseFromString(km.DistroRelease) { + switch releaseFromString(km.Distro.Release) { case Wheezy: image += "wheezy-20190228" case Jessie: @@ -159,7 +159,7 @@ func ContainerImage(km config.KernelMask) (image string) { case Stretch: image += "stretch-20220622" default: - image += km.DistroRelease + image += km.Distro.Release } return @@ -197,7 +197,7 @@ func repositories(release Release) (repos []string) { } func Runs(km config.KernelMask) (commands []string) { - release := releaseFromString(km.DistroRelease) + release := releaseFromString(km.Distro.Release) cmdf := func(f string, s ...interface{}) { commands = append(commands, fmt.Sprintf(f, s...)) @@ -288,8 +288,7 @@ func ContainerKernels(d container.Image, kcfg *config.KernelConfig) (err error) release := strings.Replace(pkgname, "linux-image-", "", -1) ki := config.KernelInfo{ - DistroType: d.DistroType, - DistroRelease: d.DistroRelease, + Distro: d.Distro, KernelVersion: path.Base(modules), KernelRelease: release, ContainerName: d.Name, @@ -332,7 +331,7 @@ func Install(km config.KernelMask, pkgname string, headers bool) (cmds []string, for _, pkg := range pkgs { found, newurl := cache.PackageURL( - km.DistroType, + km.Distro.ID, pkg.Deb.URL, ) if found { diff --git a/distro/debian/debian_test.go b/distro/debian/debian_test.go index baf0514..c19f033 100644 --- a/distro/debian/debian_test.go +++ b/distro/debian/debian_test.go @@ -5,6 +5,7 @@ import ( "testing" "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/fs" ) @@ -24,8 +25,8 @@ func TestMatch(t *testing.T) { config.Directory = tmp km := config.KernelMask{ - ReleaseMask: "3.2.0-4", - DistroRelease: "7", + ReleaseMask: "3.2.0-4", + Distro: distro.Distro{Release: "7"}, } pkgs, err := Match(km) diff --git a/distro/distro.go b/distro/distro.go new file mode 100644 index 0000000..fc489c6 --- /dev/null +++ b/distro/distro.go @@ -0,0 +1,68 @@ +package distro + +import ( + "fmt" + "strings" +) + +// ID of the distro +type ID int + +const ( + // Ubuntu https://ubuntu.com/ + Ubuntu ID = iota + // CentOS https://www.centos.org/ + CentOS + // Debian https://www.debian.org/ + Debian + // OracleLinux https://www.oracle.com/linux/ + OracleLinux +) + +var IDs = []ID{ + Ubuntu, CentOS, Debian, OracleLinux, +} + +var nameStrings = [...]string{ + "Ubuntu", + "CentOS", + "Debian", + "OracleLinux", +} + +func NewID(name string) (id ID, err error) { + err = id.UnmarshalTOML([]byte(name)) + return +} + +func (id ID) String() string { + return nameStrings[id] +} + +// UnmarshalTOML is for support github.com/naoina/toml +func (id *ID) UnmarshalTOML(data []byte) (err error) { + name := strings.Trim(string(data), `"`) + if strings.EqualFold(name, "Ubuntu") { + *id = Ubuntu + } else if strings.EqualFold(name, "CentOS") { + *id = CentOS + } else if strings.EqualFold(name, "Debian") { + *id = Debian + } else if strings.EqualFold(name, "OracleLinux") { + *id = OracleLinux + } else { + err = fmt.Errorf("distro %s is unsupported", name) + } + return +} + +// MarshalTOML is for support github.com/naoina/toml +func (id ID) MarshalTOML() (data []byte, err error) { + data = []byte(`"` + id.String() + `"`) + return +} + +type Distro struct { + ID ID + Release string +} diff --git a/distro/oraclelinux/oraclelinux.go b/distro/oraclelinux/oraclelinux.go index fe688a5..404ba56 100644 --- a/distro/oraclelinux/oraclelinux.go +++ b/distro/oraclelinux/oraclelinux.go @@ -21,7 +21,7 @@ func Runs(km config.KernelMask) (commands []string) { commands = append(commands, fmt.Sprintf(f, s...)) } - if km.DistroRelease < "6" { + if km.Distro.Release < "6" { log.Fatal().Msgf("no support for pre-EL6") } @@ -31,7 +31,7 @@ func Runs(km config.KernelMask) (commands []string) { cmdf("yum -y groupinstall 'Development Tools'") packages := "linux-firmware grubby" - if km.DistroRelease <= "7" { + if km.Distro.Release <= "7" { packages += " libdtrace-ctf" } @@ -101,7 +101,7 @@ func Install(km config.KernelMask, pkgname string, headers bool) (commands []str version = strings.Replace(pkgname, "kernel-", "", -1) } - if km.DistroRelease <= "7" { + if km.Distro.Release <= "7" { cmdf("dracut -v --add-drivers 'e1000 ext4' -f "+ "/boot/initramfs-%s.img %s", version, version) } else { diff --git a/distro/ubuntu/ubuntu.go b/distro/ubuntu/ubuntu.go index d24af95..60568ac 100644 --- a/distro/ubuntu/ubuntu.go +++ b/distro/ubuntu/ubuntu.go @@ -20,7 +20,7 @@ func Runs(km config.KernelMask) (commands []string) { commands = append(commands, fmt.Sprintf(f, s...)) } - if km.DistroRelease < "14.04" { + if km.Distro.Release < "14.04" { cmdf("sed -i 's/archive.ubuntu.com/old-releases.ubuntu.com/' " + "/etc/apt/sources.list") } @@ -29,7 +29,7 @@ func Runs(km config.KernelMask) (commands []string) { cmdf("apt-get install -y build-essential libelf-dev") cmdf("apt-get install -y wget git") - if km.DistroRelease < "14.04" { + if km.Distro.Release < "14.04" { return } diff --git a/examples/kernel-exploit/.out-of-tree.toml b/examples/kernel-exploit/.out-of-tree.toml index 0c98108..3db84f2 100644 --- a/examples/kernel-exploit/.out-of-tree.toml +++ b/examples/kernel-exploit/.out-of-tree.toml @@ -4,32 +4,27 @@ name = "out-of-tree exploit example" type = "exploit" [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "16.04" +distro = { id = "Ubuntu", release = "16.04" } release_mask = "4[.]4[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116)-.*" [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "16.04" +distro = { id = "Ubuntu", release = "16.04" } release_mask = "4[.]8[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58)-.*" [[supported_kernels]] # Can be Ubuntu/CentOS/Debian/etc. -distro_type = "Ubuntu" -distro_release = "16.04" +distro = { id = "Ubuntu", release = "16.04" } # regex for `uname -r` # See also: regex-golang.appspot.com # stupid way to generate: $ echo '4.4.0-('$(seq 44 | xargs echo | sed 's/ /|/g')')-.*' release_mask = "4[.]10[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42)-.*" [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "16.04" +distro = { id = "Ubuntu", release = "16.04" } release_mask = "4[.]11[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14)-.*" [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "16.04" +distro = { id = "Ubuntu", release = "16.04" } # equivalent for "4[.]13[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21)-.*" [supported_kernels.kernel] version = [ 4 ] diff --git a/examples/kernel-module/.out-of-tree.toml b/examples/kernel-module/.out-of-tree.toml index 96c9954..6e0c412 100644 --- a/examples/kernel-module/.out-of-tree.toml +++ b/examples/kernel-module/.out-of-tree.toml @@ -5,26 +5,22 @@ type = "module" [[supported_kernels]] # Can be Ubuntu/CentOS/Debian/etc. -distro_type = "Ubuntu" -distro_release = "16.04" +distro = { id = "Ubuntu", release = "16.04" } # regex for `uname -r` # See also: regex-golang.appspot.com release_mask = "4[.]4[.]0-70-.*" # [[supported_kernels]] may be defined unlimited number of times [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "18.04" +distro = { id = "Ubuntu", release = "18.04" } # Also you can use only one kernel release_mask = "4[.]15[.]0-(24|29)-generic" [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "18.04" +distro = { id = "Ubuntu", release = "18.04" } # Also you can use only one kernel release_mask = "4[.]15[.]0-23-generic" [[supported_kernels]] -distro_type = "CentOS" -distro_release = "7" +distro = { id = "CentOS", release = "7" } release_mask = "3[.]10[.]0-862.el7.x86_64" diff --git a/examples/preload/.out-of-tree.toml b/examples/preload/.out-of-tree.toml index 11af8ef..3145a0e 100644 --- a/examples/preload/.out-of-tree.toml +++ b/examples/preload/.out-of-tree.toml @@ -2,8 +2,7 @@ name = "out-of-tree preload" type = "module" [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "18.04" +distro = { id = "Ubuntu", release = "18.04" } release_mask = ".*" [[preload]] diff --git a/examples/script/.out-of-tree.toml b/examples/script/.out-of-tree.toml index 813ffa0..4207cce 100644 --- a/examples/script/.out-of-tree.toml +++ b/examples/script/.out-of-tree.toml @@ -6,6 +6,5 @@ type = "script" script = "script.sh" [[supported_kernels]] -distro_type = "Ubuntu" -distro_release = "22.04" +distro = { id = "Ubuntu", release = "22.04" } release_mask = ".*" diff --git a/gen.go b/gen.go index 5a55138..1d34b8a 100644 --- a/gen.go +++ b/gen.go @@ -10,6 +10,7 @@ import ( "github.com/naoina/toml" "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/distro" ) type GenCmd struct { @@ -32,9 +33,12 @@ func genConfig(at config.ArtifactType) (err error) { Type: at, } a.SupportedKernels = append(a.SupportedKernels, config.KernelMask{ - DistroType: config.Ubuntu, - DistroRelease: "18.04", - ReleaseMask: ".*", + Distro: distro.Distro{ID: distro.Ubuntu, Release: "18.04"}, + ReleaseMask: ".*", + }) + a.SupportedKernels = append(a.SupportedKernels, config.KernelMask{ + Distro: distro.Distro{ID: distro.Debian, Release: "8"}, + ReleaseMask: ".*", }) a.Preload = append(a.Preload, config.PreloadModule{ Repo: "Repo name (e.g. https://github.com/openwall/lkrg)", diff --git a/kernel.go b/kernel.go index 3619a80..e2ce870 100644 --- a/kernel.go +++ b/kernel.go @@ -13,6 +13,7 @@ import ( "code.dumpstack.io/tools/out-of-tree/config" "code.dumpstack.io/tools/out-of-tree/container" + "code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/kernel" ) @@ -46,7 +47,7 @@ func (cmd *KernelListCmd) Run(g *Globals) (err error) { } for _, k := range kcfg.Kernels { - fmt.Println(k.DistroType, k.DistroRelease, k.KernelRelease) + fmt.Println(k.Distro.ID, k.Distro.Release, k.KernelRelease) } return @@ -58,15 +59,14 @@ type KernelListRemoteCmd struct { } func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) { - distroType, err := config.NewDistroType(cmd.Distro) + distroType, err := distro.NewID(cmd.Distro) if err != nil { return } km := config.KernelMask{ - DistroType: distroType, - DistroRelease: cmd.Ver, - ReleaseMask: ".*", + Distro: distro.Distro{ID: distroType, Release: cmd.Ver}, + ReleaseMask: ".*", } _, err = kernel.GenRootfsImage(container.Image{Name: km.DockerName()}, false) @@ -108,7 +108,7 @@ func (cmd KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) { kernel.SetSigintHandler(&shutdown) for _, sk := range ka.SupportedKernels { - if sk.DistroRelease == "" { + if sk.Distro.Release == "" { err = errors.New("Please set distro_release") return } @@ -141,7 +141,7 @@ type KernelGenallCmd struct { } func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) { - distroType, err := config.NewDistroType(cmd.Distro) + distroType, err := distro.NewID(cmd.Distro) if err != nil { return } @@ -150,9 +150,8 @@ func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) { kernel.SetSigintHandler(&shutdown) km := config.KernelMask{ - DistroType: distroType, - DistroRelease: cmd.Ver, - ReleaseMask: ".*", + Distro: distro.Distro{ID: distroType, Release: cmd.Ver}, + ReleaseMask: ".*", } err = kernel.GenerateKernels(km, g.Config.Docker.Registry, @@ -179,7 +178,7 @@ type KernelInstallCmd struct { } func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) { - distroType, err := config.NewDistroType(cmd.Distro) + distroType, err := distro.NewID(cmd.Distro) if err != nil { return } @@ -188,9 +187,8 @@ func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) { kernel.SetSigintHandler(&shutdown) km := config.KernelMask{ - DistroType: distroType, - DistroRelease: cmd.Ver, - ReleaseMask: cmd.Kernel, + Distro: distro.Distro{ID: distroType, Release: cmd.Ver}, + ReleaseMask: cmd.Kernel, } err = kernel.GenerateKernels(km, g.Config.Docker.Registry, diff --git a/kernel/kernel.go b/kernel/kernel.go index 3492266..f7de235 100644 --- a/kernel/kernel.go +++ b/kernel/kernel.go @@ -23,6 +23,7 @@ import ( "code.dumpstack.io/tools/out-of-tree/cache" "code.dumpstack.io/tools/out-of-tree/config" "code.dumpstack.io/tools/out-of-tree/container" + "code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/distro/centos" "code.dumpstack.io/tools/out-of-tree/distro/debian" "code.dumpstack.io/tools/out-of-tree/distro/oraclelinux" @@ -32,15 +33,15 @@ import ( func MatchPackages(km config.KernelMask) (pkgs []string, err error) { // TODO interface for kernels match - switch km.DistroType { - case config.Ubuntu: + switch km.Distro.ID { + case distro.Ubuntu: pkgs, err = ubuntu.Match(km) - case config.OracleLinux, config.CentOS: + case distro.OracleLinux, distro.CentOS: pkgs, err = oraclelinux.Match(km) - case config.Debian: + case distro.Debian: pkgs, err = debian.Match(km) default: - err = fmt.Errorf("%s not yet supported", km.DistroType.String()) + err = fmt.Errorf("%s not yet supported", km.Distro.ID.String()) } return } @@ -83,7 +84,7 @@ func GenerateBaseDockerImage(registry string, commands []config.DockerCommand, if fs.PathExists(dockerPath) && string(rawOutput) != "" { log.Debug().Msgf("Base image for %s:%s found", - sk.DistroType.String(), sk.DistroRelease) + sk.Distro.ID.String(), sk.Distro.Release) if !forceUpdate { return } else { @@ -92,7 +93,7 @@ func GenerateBaseDockerImage(registry string, commands []config.DockerCommand, } log.Debug().Msgf("Base image for %s:%s not found, start generating", - sk.DistroType.String(), sk.DistroRelease) + sk.Distro.ID.String(), sk.Distro.Release) os.MkdirAll(imagePath, os.ModePerm) d += "FROM " @@ -100,13 +101,13 @@ func GenerateBaseDockerImage(registry string, commands []config.DockerCommand, d += registry + "/" } - switch sk.DistroType { - case config.Debian: + switch sk.Distro.ID { + case distro.Debian: d += debian.ContainerImage(sk) + "\n" default: d += fmt.Sprintf("%s:%s\n", - strings.ToLower(sk.DistroType.String()), - sk.DistroRelease) + strings.ToLower(sk.Distro.ID.String()), + sk.Distro.Release) } for _, c := range commands { @@ -114,29 +115,29 @@ func GenerateBaseDockerImage(registry string, commands []config.DockerCommand, } // TODO container runs/envs interface - switch sk.DistroType { - case config.Ubuntu: + switch sk.Distro.ID { + case distro.Ubuntu: for _, e := range ubuntu.Envs(sk) { d += "ENV " + e + "\n" } for _, c := range ubuntu.Runs(sk) { d += "RUN " + c + "\n" } - case config.CentOS: + case distro.CentOS: for _, e := range centos.Envs(sk) { d += "ENV " + e + "\n" } for _, c := range centos.Runs(sk) { d += "RUN " + c + "\n" } - case config.OracleLinux: + case distro.OracleLinux: for _, e := range oraclelinux.Envs(sk) { d += "ENV " + e + "\n" } for _, c := range oraclelinux.Runs(sk) { d += "RUN " + c + "\n" } - case config.Debian: + case distro.Debian: for _, e := range debian.Envs(sk) { d += "ENV " + e + "\n" } @@ -144,7 +145,7 @@ func GenerateBaseDockerImage(registry string, commands []config.DockerCommand, d += "RUN " + c + "\n" } default: - err = fmt.Errorf("%s not yet supported", sk.DistroType.String()) + err = fmt.Errorf("%s not yet supported", sk.Distro.ID.String()) return } @@ -163,21 +164,21 @@ func GenerateBaseDockerImage(registry string, commands []config.DockerCommand, output, err := c.Build(imagePath) if err != nil { log.Error().Err(err).Msgf("Base image for %s:%s generating error", - sk.DistroType.String(), sk.DistroRelease) + sk.Distro.ID.String(), sk.Distro.Release) log.Fatal().Msg(output) return } log.Debug().Msgf("Base image for %s:%s generating success", - sk.DistroType.String(), sk.DistroRelease) + sk.Distro.ID.String(), sk.Distro.Release) return } func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (err error) { slog := log.With(). - Str("distro_type", sk.DistroType.String()). - Str("distro_release", sk.DistroRelease). + Str("distro_type", sk.Distro.ID.String()). + Str("distro_release", sk.Distro.Release). Str("pkg", pkgname). Logger() @@ -188,7 +189,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e searchdir := c.Volumes.LibModules - if sk.DistroType == config.Debian { + if sk.Distro.ID == distro.Debian { // TODO We need some kind of API for that searchdir = config.Dir("volumes", sk.DockerName()) } @@ -209,7 +210,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e } } - if sk.DistroType == config.Debian { + if sk.Distro.ID == distro.Debian { // Debian has different kernels (package version) by the // same name (ABI), so we need to separate /boot c.Volumes = debian.Volumes(sk, pkgname) @@ -226,8 +227,8 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e var commands []string // TODO install/cleanup kernel interface - switch sk.DistroType { - case config.Ubuntu: + switch sk.Distro.ID { + case distro.Ubuntu: commands, err = ubuntu.Install(sk, pkgname, headers) if err != nil { return @@ -237,7 +238,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e ubuntu.Cleanup(sk, pkgname) } }() - case config.OracleLinux, config.CentOS: + case distro.OracleLinux, distro.CentOS: commands, err = oraclelinux.Install(sk, pkgname, headers) if err != nil { return @@ -247,7 +248,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e oraclelinux.Cleanup(sk, pkgname) } }() - case config.Debian: + case distro.Debian: commands, err = debian.Install(sk, pkgname, headers) if err != nil { return @@ -258,7 +259,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e } }() default: - err = fmt.Errorf("%s not yet supported", sk.DistroType.String()) + err = fmt.Errorf("%s not yet supported", sk.Distro.ID.String()) return } @@ -273,7 +274,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e cmd += " && cp -r /boot /target/" cmd += " && cp -r /lib/modules /target/lib/" - if sk.DistroType == config.Debian { + if sk.Distro.ID == distro.Debian { cmd += " && cp -rL /usr/src /target/usr/" } else { cmd += " && cp -r /usr/src /target/usr/" @@ -444,8 +445,7 @@ func listContainersKernels(dii container.Image, newkcfg *config.KernelConfig, } ki := config.KernelInfo{ - DistroType: dii.DistroType, - DistroRelease: dii.DistroRelease, + Distro: dii.Distro, KernelVersion: krel.Name(), KernelRelease: krel.Name(), ContainerName: dii.Name, diff --git a/kernel/kernel_linux.go b/kernel/kernel_linux.go index 269ce28..c32e229 100644 --- a/kernel/kernel_linux.go +++ b/kernel/kernel_linux.go @@ -17,6 +17,7 @@ import ( "code.dumpstack.io/tools/out-of-tree/config" "code.dumpstack.io/tools/out-of-tree/container" + "code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/fs" ) @@ -24,7 +25,7 @@ func genHostKernels(download bool) (kcfg config.KernelConfig, err error) { si := sysinfo.SysInfo{} si.GetSysInfo() - distroType, err := config.NewDistroType(si.OS.Vendor) + distroType, err := distro.NewID(si.OS.Vendor) if err != nil { return } @@ -47,8 +48,10 @@ func genHostKernels(download bool) (kcfg config.KernelConfig, err error) { // only for compatibility, docker is not really used dii := container.Image{ Name: config.KernelMask{ - DistroType: distroType, - DistroRelease: si.OS.Version, + Distro: distro.Distro{ + ID: distroType, + Release: si.OS.Version, + }, }.DockerName(), } @@ -74,8 +77,11 @@ func genHostKernels(download bool) (kcfg config.KernelConfig, err error) { } ki := config.KernelInfo{ - DistroType: distroType, - DistroRelease: si.OS.Version, + Distro: distro.Distro{ + ID: distroType, + Release: si.OS.Version, + }, + KernelRelease: krel, KernelSource: "/lib/modules/" + krel + "/build", diff --git a/log.go b/log.go index 03febad..f37b487 100644 --- a/log.go +++ b/log.go @@ -114,7 +114,7 @@ func (cmd *LogDumpCmd) Run(g *Globals) (err error) { fmt.Println("Name:", l.Name) fmt.Println() - fmt.Println("Distro:", l.DistroType.String(), l.DistroRelease) + fmt.Println("Distro:", l.Distro.ID.String(), l.Distro.Release) fmt.Println("Kernel:", l.KernelRelease) fmt.Println() @@ -208,8 +208,8 @@ func (cmd *LogMarkdownCmd) Run(g *Globals) (err error) { } func logLogEntry(l logEntry) { - distroInfo := fmt.Sprintf("%s-%s {%s}", l.DistroType, - l.DistroRelease, l.KernelRelease) + distroInfo := fmt.Sprintf("%s-%s {%s}", l.Distro.ID, + l.Distro.Release, l.KernelRelease) artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name) @@ -263,17 +263,17 @@ func getStats(db *sql.DB, path, tag string) ( distros = make(map[string]map[string]map[string]runstat) for _, l := range les { - _, ok := distros[l.DistroType.String()] + _, ok := distros[l.Distro.ID.String()] if !ok { - distros[l.DistroType.String()] = make(map[string]map[string]runstat) + distros[l.Distro.ID.String()] = make(map[string]map[string]runstat) } - _, ok = distros[l.DistroType.String()][l.DistroRelease] + _, ok = distros[l.Distro.ID.String()][l.Distro.Release] if !ok { - distros[l.DistroType.String()][l.DistroRelease] = make(map[string]runstat) + distros[l.Distro.ID.String()][l.Distro.Release] = make(map[string]runstat) } - rs := distros[l.DistroType.String()][l.DistroRelease][l.KernelRelease] + rs := distros[l.Distro.ID.String()][l.Distro.Release][l.KernelRelease] rs.All++ if l.Build.Ok { @@ -292,7 +292,7 @@ func getStats(db *sql.DB, path, tag string) ( rs.Timeout++ } - distros[l.DistroType.String()][l.DistroRelease][l.KernelRelease] = rs + distros[l.Distro.ID.String()][l.Distro.Release][l.KernelRelease] = rs } return diff --git a/pew.go b/pew.go index ecd64cb..c3d1613 100644 --- a/pew.go +++ b/pew.go @@ -25,6 +25,7 @@ import ( "code.dumpstack.io/tools/out-of-tree/config" "code.dumpstack.io/tools/out-of-tree/container" + "code.dumpstack.io/tools/out-of-tree/distro" "code.dumpstack.io/tools/out-of-tree/fs" "code.dumpstack.io/tools/out-of-tree/qemu" ) @@ -429,8 +430,8 @@ func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo, log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka) } - path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.DistroType, - ki.DistroRelease, ki.KernelRelease) + path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.Distro.ID, + ki.Distro.Release, ki.KernelRelease) if ka.Type != config.KernelExploit { path += ".ko" } @@ -571,8 +572,8 @@ func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup, logfile := fmt.Sprintf("logs/%s/%s-%s-%s.log", cmd.Tag, - ki.DistroType.String(), - ki.DistroRelease, + ki.Distro.ID.String(), + ki.Distro.Release, ki.KernelRelease, ) f, err := os.Create(logfile) @@ -602,8 +603,8 @@ func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup, } slog = slog.With().Timestamp(). - Str("distro_type", ki.DistroType.String()). - Str("distro_release", ki.DistroRelease). + Str("distro_type", ki.Distro.ID.String()). + Str("distro_release", ki.Distro.Release). Str("kernel", ki.KernelRelease). Logger() @@ -837,25 +838,19 @@ func kernelMask(kernel string) (km config.KernelMask, err error) { return } - dt, err := config.NewDistroType(parts[0]) + dt, err := distro.NewID(parts[0]) if err != nil { return } - km = config.KernelMask{DistroType: dt, ReleaseMask: parts[1]} + km = config.KernelMask{Distro: distro.Distro{ID: dt}, ReleaseMask: parts[1]} return } func genAllKernels() (sk []config.KernelMask, err error) { - for _, dType := range config.DistroTypeStrings { - var dt config.DistroType - dt, err = config.NewDistroType(dType) - if err != nil { - return - } - + for _, id := range distro.IDs { sk = append(sk, config.KernelMask{ - DistroType: dt, + Distro: distro.Distro{ID: id}, ReleaseMask: ".*", }) } diff --git a/preload.go b/preload.go index 1edeac1..4706bf7 100644 --- a/preload.go +++ b/preload.go @@ -101,9 +101,9 @@ func buildPreload(workPath, tmp string, ki config.KernelInfo, ka.SourcePath = workPath - km := config.KernelMask{DistroType: ki.DistroType, - DistroRelease: ki.DistroRelease, - ReleaseMask: ki.KernelRelease, + km := config.KernelMask{ + Distro: ki.Distro, + ReleaseMask: ki.KernelRelease, } ka.SupportedKernels = []config.KernelMask{km}