From 1ffd68601cf2646a23f75923abf300f49b57727d Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Tue, 20 Aug 2019 09:09:38 +0000 Subject: [PATCH] Remove bootstrap, download images on-demand --- CHANGELOG.md | 6 ++++ README.md | 1 - bootstrap.config.go | 7 ---- bootstrap.go | 83 ------------------------------------------- docs/installation.rst | 4 --- images.config.go | 7 ++++ images.go | 82 ++++++++++++++++++++++++++++++++++++++++++ kernel.go | 68 ++++++++++++++++++++++++----------- main.go | 28 ++++++++------- pack.go | 4 +-- 10 files changed, 160 insertions(+), 130 deletions(-) delete mode 100644 bootstrap.config.go delete mode 100644 bootstrap.go create mode 100644 images.config.go create mode 100644 images.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7f82e..56c5ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,9 @@ - Added ability to change amount of memory/CPUs and set qemu timeout in artifact definition (`.out-of-tree.toml`). +- Now images downloading while `kernel autogen`, bootstrap is not + required anymore. + ### Changed - Now if there's no base image found — out-of-tree will try to use @@ -79,6 +82,9 @@ - *Kernel factory* is removed completely in favor of incremental Dockerfiles. +- `bootstrap` is not doing anything anymore. It'll be removed in next + release. + ### Fixed - Command `timeout` is not required anymore. diff --git a/README.md b/README.md index 9e137ef..edaf1f6 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,6 @@ Also check out [docker post-installation steps](https://docs.docker.com/install/ ## Build from source $ go get -u code.dumpstack.io/tools/out-of-tree - $ out-of-tree bootstrap Then you can check it on kernel module example: diff --git a/bootstrap.config.go b/bootstrap.config.go deleted file mode 100644 index 585e327..0000000 --- a/bootstrap.config.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2018 Mikhail Klementev. All rights reserved. -// Use of this source code is governed by a AGPLv3 license -// (or later) that can be found in the LICENSE file. - -package main - -const imagesURL = "https://github.com/jollheef/out-of-tree/releases/download/v0.2/images.tar.gz" diff --git a/bootstrap.go b/bootstrap.go deleted file mode 100644 index 018dd70..0000000 --- a/bootstrap.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Mikhail Klementev. All rights reserved. -// Use of this source code is governed by a AGPLv3 license -// (or later) that can be found in the LICENSE file. - -package main - -import ( - "io" - "io/ioutil" - "log" - "net/http" - "os" - "os/exec" - "os/user" -) - -// inspired by Edd Turtle code -func downloadFile(filepath string, url string) (err error) { - out, err := os.Create(filepath) - if err != nil { - return - } - defer out.Close() - - resp, err := http.Get(url) - if err != nil { - return - } - defer resp.Body.Close() - - _, err = io.Copy(out, resp.Body) - return -} - -func unpackTar(archive, destination string) (err error) { - cmd := exec.Command("tar", "xf", archive) - cmd.Dir = destination + "/" - - rawOutput, err := cmd.CombinedOutput() - if err != nil { - // I don't like when some errors printed inside - // So if you know way to do it better - FIXME please - log.Println("Unpack images error:", string(rawOutput), err) - return - } - - return -} - -func bootstrapHandler() (err error) { - log.Println("Download images...") - - usr, err := user.Current() - if err != nil { - return - } - - imagesPath := usr.HomeDir + "/.out-of-tree/images/" - os.MkdirAll(imagesPath, os.ModePerm) - - tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_") - if err != nil { - log.Println("Temporary directory creation error:", err) - return - } - defer os.RemoveAll(tmp) - - imagesArchive := tmp + "/images.tar.gz" - - err = downloadFile(imagesArchive, imagesURL) - if err != nil { - log.Println("Download file error:", err) - return - } - - err = unpackTar(imagesArchive, imagesPath) - if err != nil { - log.Println("Unpack images error:", err) - } - - log.Println("Success!") - return -} diff --git a/docs/installation.rst b/docs/installation.rst index 11de760..ed1ec56 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -55,10 +55,6 @@ Build *out-of-tree*:: security implications. Check *Docker* documentation for more information. -Bootstrap:: - - $ out-of-tree bootstrap - Test that everything works:: $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/examples/kernel-exploit diff --git a/images.config.go b/images.config.go new file mode 100644 index 0000000..eb46b7e --- /dev/null +++ b/images.config.go @@ -0,0 +1,7 @@ +// Copyright 2019 Mikhail Klementev. All rights reserved. +// Use of this source code is governed by a AGPLv3 license +// (or later) that can be found in the LICENSE file. + +package main + +const imagesBaseURL = "https://out-of-tree.fra1.digitaloceanspaces.com/1.0.0/" diff --git a/images.go b/images.go new file mode 100644 index 0000000..a8de113 --- /dev/null +++ b/images.go @@ -0,0 +1,82 @@ +// Copyright 2019 Mikhail Klementev. All rights reserved. +// Use of this source code is governed by a AGPLv3 license +// (or later) that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" +) + +// inspired by Edd Turtle code +func downloadFile(filepath string, url string) (err error) { + out, err := os.Create(filepath) + if err != nil { + return + } + defer out.Close() + + resp, err := http.Get(url) + if err != nil { + return + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + break + case http.StatusForbidden, http.StatusNotFound: + err = fmt.Errorf("Cannot download %s. It looks like you need "+ + "to generate it manually and place it "+ + "to ~/.out-of-tree/images/. "+ + "Check documentation for additional information.", url) + return + default: + err = fmt.Errorf("Something weird happens while "+ + "download file: %d", resp.StatusCode) + return + } + + _, err = io.Copy(out, resp.Body) + return +} + +func unpackTar(archive, destination string) (err error) { + // NOTE: If you're change anything in tar command please check also + // BSD tar (or if you're using macOS, do not forget to check GNU Tar) + // Also make sure that sparse files are extracting correctly + cmd := exec.Command("tar", "-Sxf", archive) + cmd.Dir = destination + "/" + + rawOutput, err := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("%v: %s", err, rawOutput) + return + } + + return +} + +func downloadImage(path, file string) (err error) { + tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_") + if err != nil { + return + } + defer os.RemoveAll(tmp) + + archive := tmp + "/" + file + ".tar.gz" + url := imagesBaseURL + file + ".tar.gz" + + err = downloadFile(archive, url) + if err != nil { + return + } + + err = unpackTar(archive, path) + return +} diff --git a/kernel.go b/kernel.go index 11d289d..17ebc4a 100644 --- a/kernel.go +++ b/kernel.go @@ -259,13 +259,24 @@ func genInitrdPath(files []os.FileInfo, kname string) string { return "unknown" } -func genRootfsImage(d dockerImageInfo) string { +func genRootfsImage(d dockerImageInfo, download bool) (rootfs string, err error) { usr, err := user.Current() if err != nil { - return fmt.Sprintln(err) + return } imageFile := d.ContainerName + ".img" - return usr.HomeDir + "/.out-of-tree/images/" + imageFile + + imagesPath := usr.HomeDir + "/.out-of-tree/images/" + os.MkdirAll(imagesPath, os.ModePerm) + + rootfs = imagesPath + imageFile + if !exists(rootfs) { + if download { + log.Println(imageFile, "not exists, start downloading...") + err = downloadImage(imagesPath, imageFile) + } + } + return } type dockerImageInfo struct { @@ -309,7 +320,7 @@ func listDockerImages() (diis []dockerImageInfo, err error) { return } -func genHostKernels() (kcfg config.KernelConfig, err error) { +func genHostKernels(download bool) (kcfg config.KernelConfig, err error) { si := sysinfo.SysInfo{} si.GetSysInfo() @@ -339,6 +350,11 @@ func genHostKernels() (kcfg config.KernelConfig, err error) { }.DockerName(), } + rootfs, err := genRootfsImage(dii, download) + if err != nil { + return + } + for _, k := range strings.Fields(string(rawOutput)) { ki := config.KernelInfo{ DistroType: distroType, @@ -349,7 +365,7 @@ func genHostKernels() (kcfg config.KernelConfig, err error) { KernelPath: kernelsBase + genKernelPath(files, k), InitrdPath: kernelsBase + genInitrdPath(files, k), - RootFS: genRootfsImage(dii), + RootFS: rootfs, } vmlinux := "/usr/lib/debug/boot/vmlinux-" + k @@ -364,12 +380,12 @@ func genHostKernels() (kcfg config.KernelConfig, err error) { return } -func updateKernelsCfg(host bool) (err error) { +func updateKernelsCfg(host, download bool) (err error) { newkcfg := config.KernelConfig{} if host { // Get host kernels - newkcfg, err = genHostKernels() + newkcfg, err = genHostKernels(download) if err != nil { return } @@ -382,7 +398,7 @@ func updateKernelsCfg(host bool) (err error) { } for _, d := range dockerImages { - err = genDockerKernels(d, &newkcfg) + err = genDockerKernels(d, &newkcfg, download) if err != nil { log.Println("gen kernels", d.ContainerName, ":", err) continue @@ -419,8 +435,8 @@ func updateKernelsCfg(host bool) (err error) { return } -func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) ( - err error) { +func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig, + download bool) (err error) { name := dii.ContainerName cmd := exec.Command("docker", "run", name, "ls", "/lib/modules") @@ -440,6 +456,11 @@ func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) ( return } + rootfs, err := genRootfsImage(dii, download) + if err != nil { + return + } + for _, k := range strings.Fields(string(rawOutput)) { ki := config.KernelInfo{ DistroType: dii.DistroType, @@ -449,7 +470,7 @@ func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) ( KernelPath: kernelsBase + genKernelPath(files, k), InitrdPath: kernelsBase + genInitrdPath(files, k), - RootFS: genRootfsImage(dii), + RootFS: rootfs, } newkcfg.Kernels = append(newkcfg.Kernels, ki) } @@ -475,8 +496,15 @@ func shuffle(a []string) []string { return a } -func generateKernels(km config.KernelMask, max int64) (err error) { +func generateKernels(km config.KernelMask, max int64, download bool) (err error) { log.Println("Generating for kernel mask", km) + + _, err = genRootfsImage(dockerImageInfo{ContainerName: km.DockerName()}, + download) + if err != nil { + return + } + err = generateBaseDockerImage(km) if err != nil { return @@ -519,7 +547,7 @@ func generateKernels(km config.KernelMask, max int64) (err error) { return } -func kernelAutogenHandler(workPath string, max int64, host bool) (err error) { +func kernelAutogenHandler(workPath string, max int64, host, download bool) (err error) { ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml") if err != nil { return @@ -531,17 +559,17 @@ func kernelAutogenHandler(workPath string, max int64, host bool) (err error) { return } - err = generateKernels(sk, max) + err = generateKernels(sk, max, download) if err != nil { return } } - err = updateKernelsCfg(host) + err = updateKernelsCfg(host, download) return } -func kernelDockerRegenHandler(host bool) (err error) { +func kernelDockerRegenHandler(host, download bool) (err error) { dockerImages, err := listDockerImages() if err != nil { return @@ -579,10 +607,10 @@ func kernelDockerRegenHandler(host bool) (err error) { } } - return updateKernelsCfg(host) + return updateKernelsCfg(host, download) } -func kernelGenallHandler(distro, version string, host bool) (err error) { +func kernelGenallHandler(distro, version string, host, download bool) (err error) { distroType, err := config.NewDistroType(distro) if err != nil { return @@ -593,10 +621,10 @@ func kernelGenallHandler(distro, version string, host bool) (err error) { DistroRelease: version, ReleaseMask: ".*", } - err = generateKernels(km, kernelsAll) + err = generateKernels(km, kernelsAll, download) if err != nil { return } - return updateKernelsCfg(host) + return updateKernelsCfg(host, download) } diff --git a/main.go b/main.go index 3141486..a2bca42 100644 --- a/main.go +++ b/main.go @@ -145,6 +145,8 @@ func main() { pewTag := pewTagFlag.String() kernelCommand := app.Command("kernel", "Manipulate kernels") + kernelNoDownload := kernelCommand.Flag("no-download", + "Do not download qemu image while kernel generation").Bool() kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool() kernelListCommand := kernelCommand.Command("list", "List kernels") kernelAutogenCommand := kernelCommand.Command("autogen", @@ -187,8 +189,7 @@ func main() { nosmap := debugCommand.Flag("disable-smap", "Disable SMAP").Bool() nokpti := debugCommand.Flag("disable-kpti", "Disable KPTI").Bool() - bootstrapCommand := app.Command("bootstrap", - "Create directories && download images") + bootstrapCommand := app.Command("bootstrap", "Apparently nothing") logCommand := app.Command("log", "Logs") @@ -209,6 +210,8 @@ func main() { packCommand := app.Command("pack", "Exploit pack test") packAutogen := packCommand.Flag("autogen", "Kernel autogeneration").Bool() + packNoDownload := packCommand.Flag("no-download", + "Do not download qemu image while kernel generation").Bool() packExploitRuns := packCommand.Flag("exploit-runs", "Amount of runs of each exploit").Default("4").Int64() packKernelRuns := packCommand.Flag("kernel-runs", @@ -228,12 +231,6 @@ func main() { os.Exit(1) } - if !exists(usr.HomeDir + "/.out-of-tree/images") { - log.Println("No ~/.out-of-tree/images: " + - "Probably you need to run `out-of-tree bootstrap`" + - " for downloading basic images") - } - if !exists(usr.HomeDir + "/.out-of-tree/kernels.toml") { log.Println("No ~/.out-of-tree/kernels.toml: Probably you " + "need to run `out-of-tree kernel autogen` in " + @@ -290,11 +287,13 @@ func main() { case kernelListCommand.FullCommand(): err = kernelListHandler(kcfg) case kernelAutogenCommand.FullCommand(): - err = kernelAutogenHandler(*path, *kernelAutogenMax, *kernelUseHost) + err = kernelAutogenHandler(*path, *kernelAutogenMax, + *kernelUseHost, !*kernelNoDownload) case kernelDockerRegenCommand.FullCommand(): - err = kernelDockerRegenHandler(*kernelUseHost) + err = kernelDockerRegenHandler(*kernelUseHost, !*kernelNoDownload) case kernelGenallCommand.FullCommand(): - err = kernelGenallHandler(*distro, *version, *kernelUseHost) + err = kernelGenallHandler(*distro, *version, + *kernelUseHost, !*kernelNoDownload) case genModuleCommand.FullCommand(): err = genConfig(config.KernelModule) case genExploitCommand.FullCommand(): @@ -304,7 +303,10 @@ func main() { *dockerTimeout, *yekaslr, *yesmep, *yesmap, *yekpti, *nokaslr, *nosmep, *nosmap, *nokpti) case bootstrapCommand.FullCommand(): - err = bootstrapHandler() + fmt.Println("bootstrap is no more required, " + + "now images downloading on-demand") + fmt.Println("please, remove it from any automation scripts, " + + "because it'll be removed in the next release") case logQueryCommand.FullCommand(): err = logHandler(db, *path, *logTag, *logNum, *logRate) case logDumpCommand.FullCommand(): @@ -315,7 +317,7 @@ func main() { err = logMarkdownHandler(db, *path, *logMarkdownTag) case packCommand.FullCommand(): err = packHandler(db, *path, kcfg, *packAutogen, - *packExploitRuns, *packKernelRuns) + !*packNoDownload, *packExploitRuns, *packKernelRuns) } if err != nil { diff --git a/pack.go b/pack.go index be68404..fab0f0d 100644 --- a/pack.go +++ b/pack.go @@ -16,7 +16,7 @@ import ( ) func packHandler(db *sql.DB, path string, kcfg config.KernelConfig, - autogen bool, exploitRuns, kernelRuns int64) (err error) { + autogen, download bool, exploitRuns, kernelRuns int64) (err error) { dockerTimeout := time.Minute qemuTimeout := time.Minute @@ -39,7 +39,7 @@ func packHandler(db *sql.DB, path string, kcfg config.KernelConfig, if autogen { var perRegex int64 = 1 - err = kernelAutogenHandler(workPath, perRegex, false) + err = kernelAutogenHandler(workPath, perRegex, false, download) if err != nil { return }