Remove bootstrap, download images on-demand

CHANGELOG.md

@@ -53,6 +53,9 @@
 - Added ability to change amount of memory/CPUs and set qemu timeout
   in artifact definition (`.out-of-tree.toml`).
 
+- Now images downloading while `kernel autogen`, bootstrap is not
+  required anymore.
+
 ### Changed
 
 - Now if there's no base image found — out-of-tree will try to use
@@ -79,6 +82,9 @@
 - *Kernel factory* is removed completely in favor of incremental
   Dockerfiles.
 
+- `bootstrap` is not doing anything anymore. It'll be removed in next
+  release.
+
 ### Fixed
 
 - Command `timeout` is not required anymore.

README.md

@@ -39,7 +39,6 @@ Also check out [docker post-installation steps](https://docs.docker.com/install/
 ## Build from source
 
     $ go get -u code.dumpstack.io/tools/out-of-tree
-    $ out-of-tree bootstrap
 
 Then you can check it on kernel module example:
 

bootstrap.config.go

@@ -1,7 +0,0 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
-// Use of this source code is governed by a AGPLv3 license
-// (or later) that can be found in the LICENSE file.
-
-package main
-
-const imagesURL = "https://github.com/jollheef/out-of-tree/releases/download/v0.2/images.tar.gz"

bootstrap.go

@@ -1,83 +0,0 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
-// Use of this source code is governed by a AGPLv3 license
-// (or later) that can be found in the LICENSE file.
-
-package main
-
-import (
-	"io"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"os/exec"
-	"os/user"
-)
-
-// inspired by Edd Turtle code
-func downloadFile(filepath string, url string) (err error) {
-	out, err := os.Create(filepath)
-	if err != nil {
-		return
-	}
-	defer out.Close()
-
-	resp, err := http.Get(url)
-	if err != nil {
-		return
-	}
-	defer resp.Body.Close()
-
-	_, err = io.Copy(out, resp.Body)
-	return
-}
-
-func unpackTar(archive, destination string) (err error) {
-	cmd := exec.Command("tar", "xf", archive)
-	cmd.Dir = destination + "/"
-
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		// I don't like when some errors printed inside
-		// So if you know way to do it better - FIXME please
-		log.Println("Unpack images error:", string(rawOutput), err)
-		return
-	}
-
-	return
-}
-
-func bootstrapHandler() (err error) {
-	log.Println("Download images...")
-
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-
-	imagesPath := usr.HomeDir + "/.out-of-tree/images/"
-	os.MkdirAll(imagesPath, os.ModePerm)
-
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
-	if err != nil {
-		log.Println("Temporary directory creation error:", err)
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	imagesArchive := tmp + "/images.tar.gz"
-
-	err = downloadFile(imagesArchive, imagesURL)
-	if err != nil {
-		log.Println("Download file error:", err)
-		return
-	}
-
-	err = unpackTar(imagesArchive, imagesPath)
-	if err != nil {
-		log.Println("Unpack images error:", err)
-	}
-
-	log.Println("Success!")
-	return
-}

docs/installation.rst

@@ -55,10 +55,6 @@ Build *out-of-tree*::
   security implications. Check *Docker* documentation for more
   information.
 
-Bootstrap::
-
-  $ out-of-tree bootstrap
-
 Test that everything works::
 
   $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/examples/kernel-exploit

images.config.go

@@ -0,0 +1,7 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+const imagesBaseURL = "https://out-of-tree.fra1.digitaloceanspaces.com/1.0.0/"

images.go

@@ -0,0 +1,82 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+)
+
+// inspired by Edd Turtle code
+func downloadFile(filepath string, url string) (err error) {
+	out, err := os.Create(filepath)
+	if err != nil {
+		return
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		break
+	case http.StatusForbidden, http.StatusNotFound:
+		err = fmt.Errorf("Cannot download %s. It looks like you need "+
+			"to generate it manually and place it "+
+			"to ~/.out-of-tree/images/. "+
+			"Check documentation for additional information.", url)
+		return
+	default:
+		err = fmt.Errorf("Something weird happens while "+
+			"download file: %d", resp.StatusCode)
+		return
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	return
+}
+
+func unpackTar(archive, destination string) (err error) {
+	// NOTE: If you're change anything in tar command please check also
+	// BSD tar (or if you're using macOS, do not forget to check GNU Tar)
+	// Also make sure that sparse files are extracting correctly
+	cmd := exec.Command("tar", "-Sxf", archive)
+	cmd.Dir = destination + "/"
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		err = fmt.Errorf("%v: %s", err, rawOutput)
+		return
+	}
+
+	return
+}
+
+func downloadImage(path, file string) (err error) {
+	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	archive := tmp + "/" + file + ".tar.gz"
+	url := imagesBaseURL + file + ".tar.gz"
+
+	err = downloadFile(archive, url)
+	if err != nil {
+		return
+	}
+
+	err = unpackTar(archive, path)
+	return
+}

kernel.go

@@ -259,13 +259,24 @@ func genInitrdPath(files []os.FileInfo, kname string) string {
 	return "unknown"
 }
 
-func genRootfsImage(d dockerImageInfo) string {
+func genRootfsImage(d dockerImageInfo, download bool) (rootfs string, err error) {
 	usr, err := user.Current()
 	if err != nil {
-		return fmt.Sprintln(err)
+		return
 	}
 	imageFile := d.ContainerName + ".img"
-	return usr.HomeDir + "/.out-of-tree/images/" + imageFile
+
+	imagesPath := usr.HomeDir + "/.out-of-tree/images/"
+	os.MkdirAll(imagesPath, os.ModePerm)
+
+	rootfs = imagesPath + imageFile
+	if !exists(rootfs) {
+		if download {
+			log.Println(imageFile, "not exists, start downloading...")
+			err = downloadImage(imagesPath, imageFile)
+		}
+	}
+	return
 }
 
 type dockerImageInfo struct {
@@ -309,7 +320,7 @@ func listDockerImages() (diis []dockerImageInfo, err error) {
 	return
 }
 
-func genHostKernels() (kcfg config.KernelConfig, err error) {
+func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
 	si := sysinfo.SysInfo{}
 	si.GetSysInfo()
 
@@ -339,6 +350,11 @@ func genHostKernels() (kcfg config.KernelConfig, err error) {
 		}.DockerName(),
 	}
 
+	rootfs, err := genRootfsImage(dii, download)
+	if err != nil {
+		return
+	}
+
 	for _, k := range strings.Fields(string(rawOutput)) {
 		ki := config.KernelInfo{
 			DistroType:    distroType,
@@ -349,7 +365,7 @@ func genHostKernels() (kcfg config.KernelConfig, err error) {
 
 			KernelPath: kernelsBase + genKernelPath(files, k),
 			InitrdPath: kernelsBase + genInitrdPath(files, k),
-			RootFS:     genRootfsImage(dii),
+			RootFS:     rootfs,
 		}
 
 		vmlinux := "/usr/lib/debug/boot/vmlinux-" + k
@@ -364,12 +380,12 @@ func genHostKernels() (kcfg config.KernelConfig, err error) {
 	return
 }
 
-func updateKernelsCfg(host bool) (err error) {
+func updateKernelsCfg(host, download bool) (err error) {
 	newkcfg := config.KernelConfig{}
 
 	if host {
 		// Get host kernels
-		newkcfg, err = genHostKernels()
+		newkcfg, err = genHostKernels(download)
 		if err != nil {
 			return
 		}
@@ -382,7 +398,7 @@ func updateKernelsCfg(host bool) (err error) {
 	}
 
 	for _, d := range dockerImages {
-		err = genDockerKernels(d, &newkcfg)
+		err = genDockerKernels(d, &newkcfg, download)
 		if err != nil {
 			log.Println("gen kernels", d.ContainerName, ":", err)
 			continue
@@ -419,8 +435,8 @@ func updateKernelsCfg(host bool) (err error) {
 	return
 }
 
-func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) (
+func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig,
-	err error) {
+	download bool) (err error) {
 
 	name := dii.ContainerName
 	cmd := exec.Command("docker", "run", name, "ls", "/lib/modules")
@@ -440,6 +456,11 @@ func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) (
 		return
 	}
 
+	rootfs, err := genRootfsImage(dii, download)
+	if err != nil {
+		return
+	}
+
 	for _, k := range strings.Fields(string(rawOutput)) {
 		ki := config.KernelInfo{
 			DistroType:    dii.DistroType,
@@ -449,7 +470,7 @@ func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) (
 
 			KernelPath: kernelsBase + genKernelPath(files, k),
 			InitrdPath: kernelsBase + genInitrdPath(files, k),
-			RootFS:     genRootfsImage(dii),
+			RootFS:     rootfs,
 		}
 		newkcfg.Kernels = append(newkcfg.Kernels, ki)
 	}
@@ -475,8 +496,15 @@ func shuffle(a []string) []string {
 	return a
 }
 
-func generateKernels(km config.KernelMask, max int64) (err error) {
+func generateKernels(km config.KernelMask, max int64, download bool) (err error) {
 	log.Println("Generating for kernel mask", km)
+
+	_, err = genRootfsImage(dockerImageInfo{ContainerName: km.DockerName()},
+		download)
+	if err != nil {
+		return
+	}
+
 	err = generateBaseDockerImage(km)
 	if err != nil {
 		return
@@ -519,7 +547,7 @@ func generateKernels(km config.KernelMask, max int64) (err error) {
 	return
 }
 
-func kernelAutogenHandler(workPath string, max int64, host bool) (err error) {
+func kernelAutogenHandler(workPath string, max int64, host, download bool) (err error) {
 	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
 	if err != nil {
 		return
@@ -531,17 +559,17 @@ func kernelAutogenHandler(workPath string, max int64, host bool) (err error) {
 			return
 		}
 
-		err = generateKernels(sk, max)
+		err = generateKernels(sk, max, download)
 		if err != nil {
 			return
 		}
 	}
 
-	err = updateKernelsCfg(host)
+	err = updateKernelsCfg(host, download)
 	return
 }
 
-func kernelDockerRegenHandler(host bool) (err error) {
+func kernelDockerRegenHandler(host, download bool) (err error) {
 	dockerImages, err := listDockerImages()
 	if err != nil {
 		return
@@ -579,10 +607,10 @@ func kernelDockerRegenHandler(host bool) (err error) {
 		}
 	}
 
-	return updateKernelsCfg(host)
+	return updateKernelsCfg(host, download)
 }
 
-func kernelGenallHandler(distro, version string, host bool) (err error) {
+func kernelGenallHandler(distro, version string, host, download bool) (err error) {
 	distroType, err := config.NewDistroType(distro)
 	if err != nil {
 		return
@@ -593,10 +621,10 @@ func kernelGenallHandler(distro, version string, host bool) (err error) {
 		DistroRelease: version,
 		ReleaseMask:   ".*",
 	}
-	err = generateKernels(km, kernelsAll)
+	err = generateKernels(km, kernelsAll, download)
 	if err != nil {
 		return
 	}
 
-	return updateKernelsCfg(host)
+	return updateKernelsCfg(host, download)
 }

main.go

@@ -145,6 +145,8 @@ func main() {
 	pewTag := pewTagFlag.String()
 
 	kernelCommand := app.Command("kernel", "Manipulate kernels")
+	kernelNoDownload := kernelCommand.Flag("no-download",
+		"Do not download qemu image while kernel generation").Bool()
 	kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool()
 	kernelListCommand := kernelCommand.Command("list", "List kernels")
 	kernelAutogenCommand := kernelCommand.Command("autogen",
@@ -187,8 +189,7 @@ func main() {
 	nosmap := debugCommand.Flag("disable-smap", "Disable SMAP").Bool()
 	nokpti := debugCommand.Flag("disable-kpti", "Disable KPTI").Bool()
 
-	bootstrapCommand := app.Command("bootstrap",
+	bootstrapCommand := app.Command("bootstrap", "Apparently nothing")
-		"Create directories && download images")
 
 	logCommand := app.Command("log", "Logs")
 
@@ -209,6 +210,8 @@ func main() {
 
 	packCommand := app.Command("pack", "Exploit pack test")
 	packAutogen := packCommand.Flag("autogen", "Kernel autogeneration").Bool()
+	packNoDownload := packCommand.Flag("no-download",
+		"Do not download qemu image while kernel generation").Bool()
 	packExploitRuns := packCommand.Flag("exploit-runs",
 		"Amount of runs of each exploit").Default("4").Int64()
 	packKernelRuns := packCommand.Flag("kernel-runs",
@@ -228,12 +231,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	if !exists(usr.HomeDir + "/.out-of-tree/images") {
-		log.Println("No ~/.out-of-tree/images: " +
-			"Probably you need to run `out-of-tree bootstrap`" +
-			" for downloading basic images")
-	}
-
 	if !exists(usr.HomeDir + "/.out-of-tree/kernels.toml") {
 		log.Println("No ~/.out-of-tree/kernels.toml: Probably you " +
 			"need to run `out-of-tree kernel autogen` in " +
@@ -290,11 +287,13 @@ func main() {
 	case kernelListCommand.FullCommand():
 		err = kernelListHandler(kcfg)
 	case kernelAutogenCommand.FullCommand():
-		err = kernelAutogenHandler(*path, *kernelAutogenMax, *kernelUseHost)
+		err = kernelAutogenHandler(*path, *kernelAutogenMax,
+			*kernelUseHost, !*kernelNoDownload)
 	case kernelDockerRegenCommand.FullCommand():
-		err = kernelDockerRegenHandler(*kernelUseHost)
+		err = kernelDockerRegenHandler(*kernelUseHost, !*kernelNoDownload)
 	case kernelGenallCommand.FullCommand():
-		err = kernelGenallHandler(*distro, *version, *kernelUseHost)
+		err = kernelGenallHandler(*distro, *version,
+			*kernelUseHost, !*kernelNoDownload)
 	case genModuleCommand.FullCommand():
 		err = genConfig(config.KernelModule)
 	case genExploitCommand.FullCommand():
@@ -304,7 +303,10 @@ func main() {
 			*dockerTimeout, *yekaslr, *yesmep, *yesmap, *yekpti,
 			*nokaslr, *nosmep, *nosmap, *nokpti)
 	case bootstrapCommand.FullCommand():
-		err = bootstrapHandler()
+		fmt.Println("bootstrap is no more required, " +
+			"now images downloading on-demand")
+		fmt.Println("please, remove it from any automation scripts, " +
+			"because it'll be removed in the next release")
 	case logQueryCommand.FullCommand():
 		err = logHandler(db, *path, *logTag, *logNum, *logRate)
 	case logDumpCommand.FullCommand():
@@ -315,7 +317,7 @@ func main() {
 		err = logMarkdownHandler(db, *path, *logMarkdownTag)
 	case packCommand.FullCommand():
 		err = packHandler(db, *path, kcfg, *packAutogen,
-			*packExploitRuns, *packKernelRuns)
+			!*packNoDownload, *packExploitRuns, *packKernelRuns)
 	}
 
 	if err != nil {

pack.go

@@ -16,7 +16,7 @@ import (
 )
 
 func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,
-	autogen bool, exploitRuns, kernelRuns int64) (err error) {
+	autogen, download bool, exploitRuns, kernelRuns int64) (err error) {
 
 	dockerTimeout := time.Minute
 	qemuTimeout := time.Minute
@@ -39,7 +39,7 @@ func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,
 
 		if autogen {
 			var perRegex int64 = 1
-			err = kernelAutogenHandler(workPath, perRegex, false)
+			err = kernelAutogenHandler(workPath, perRegex, false, download)
 			if err != nil {
 				return
 			}