Browse Source

Remove bootstrap, download images on-demand

tags/v1.0.0
dump_stack() 7 months ago
parent
commit
1ffd68601c
Signed by: Mikhail Klementev <blame@dumpstack.io> GPG Key ID: BE44DA8C062D87DC
10 changed files with 160 additions and 130 deletions
  1. 6
    0
      CHANGELOG.md
  2. 0
    1
      README.md
  3. 0
    7
      bootstrap.config.go
  4. 0
    83
      bootstrap.go
  5. 0
    4
      docs/installation.rst
  6. 7
    0
      images.config.go
  7. 82
    0
      images.go
  8. 48
    20
      kernel.go
  9. 15
    13
      main.go
  10. 2
    2
      pack.go

+ 6
- 0
CHANGELOG.md View File

@@ -53,6 +53,9 @@
- Added ability to change amount of memory/CPUs and set qemu timeout
in artifact definition (`.out-of-tree.toml`).

- Now images downloading while `kernel autogen`, bootstrap is not
required anymore.

### Changed

- Now if there's no base image found — out-of-tree will try to use
@@ -79,6 +82,9 @@
- *Kernel factory* is removed completely in favor of incremental
Dockerfiles.

- `bootstrap` is not doing anything anymore. It'll be removed in next
release.

### Fixed

- Command `timeout` is not required anymore.

+ 0
- 1
README.md View File

@@ -39,7 +39,6 @@ Also check out [docker post-installation steps](https://docs.docker.com/install/
## Build from source

$ go get -u code.dumpstack.io/tools/out-of-tree
$ out-of-tree bootstrap

Then you can check it on kernel module example:


+ 0
- 7
bootstrap.config.go View File

@@ -1,7 +0,0 @@
// Copyright 2018 Mikhail Klementev. All rights reserved.
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.

package main

const imagesURL = "https://github.com/jollheef/out-of-tree/releases/download/v0.2/images.tar.gz"

+ 0
- 83
bootstrap.go View File

@@ -1,83 +0,0 @@
// Copyright 2018 Mikhail Klementev. All rights reserved.
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.

package main

import (
"io"
"io/ioutil"
"log"
"net/http"
"os"
"os/exec"
"os/user"
)

// inspired by Edd Turtle code
func downloadFile(filepath string, url string) (err error) {
out, err := os.Create(filepath)
if err != nil {
return
}
defer out.Close()

resp, err := http.Get(url)
if err != nil {
return
}
defer resp.Body.Close()

_, err = io.Copy(out, resp.Body)
return
}

func unpackTar(archive, destination string) (err error) {
cmd := exec.Command("tar", "xf", archive)
cmd.Dir = destination + "/"

rawOutput, err := cmd.CombinedOutput()
if err != nil {
// I don't like when some errors printed inside
// So if you know way to do it better - FIXME please
log.Println("Unpack images error:", string(rawOutput), err)
return
}

return
}

func bootstrapHandler() (err error) {
log.Println("Download images...")

usr, err := user.Current()
if err != nil {
return
}

imagesPath := usr.HomeDir + "/.out-of-tree/images/"
os.MkdirAll(imagesPath, os.ModePerm)

tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
if err != nil {
log.Println("Temporary directory creation error:", err)
return
}
defer os.RemoveAll(tmp)

imagesArchive := tmp + "/images.tar.gz"

err = downloadFile(imagesArchive, imagesURL)
if err != nil {
log.Println("Download file error:", err)
return
}

err = unpackTar(imagesArchive, imagesPath)
if err != nil {
log.Println("Unpack images error:", err)
}

log.Println("Success!")
return
}

+ 0
- 4
docs/installation.rst View File

@@ -55,10 +55,6 @@ Build *out-of-tree*::
security implications. Check *Docker* documentation for more
information.

Bootstrap::

$ out-of-tree bootstrap

Test that everything works::

$ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/examples/kernel-exploit

+ 7
- 0
images.config.go View File

@@ -0,0 +1,7 @@
// Copyright 2019 Mikhail Klementev. All rights reserved.
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.

package main

const imagesBaseURL = "https://out-of-tree.fra1.digitaloceanspaces.com/1.0.0/"

+ 82
- 0
images.go View File

@@ -0,0 +1,82 @@
// Copyright 2019 Mikhail Klementev. All rights reserved.
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.

package main

import (
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
)

// inspired by Edd Turtle code
func downloadFile(filepath string, url string) (err error) {
out, err := os.Create(filepath)
if err != nil {
return
}
defer out.Close()

resp, err := http.Get(url)
if err != nil {
return
}
defer resp.Body.Close()

switch resp.StatusCode {
case http.StatusOK:
break
case http.StatusForbidden, http.StatusNotFound:
err = fmt.Errorf("Cannot download %s. It looks like you need "+
"to generate it manually and place it "+
"to ~/.out-of-tree/images/. "+
"Check documentation for additional information.", url)
return
default:
err = fmt.Errorf("Something weird happens while "+
"download file: %d", resp.StatusCode)
return
}

_, err = io.Copy(out, resp.Body)
return
}

func unpackTar(archive, destination string) (err error) {
// NOTE: If you're change anything in tar command please check also
// BSD tar (or if you're using macOS, do not forget to check GNU Tar)
// Also make sure that sparse files are extracting correctly
cmd := exec.Command("tar", "-Sxf", archive)
cmd.Dir = destination + "/"

rawOutput, err := cmd.CombinedOutput()
if err != nil {
err = fmt.Errorf("%v: %s", err, rawOutput)
return
}

return
}

func downloadImage(path, file string) (err error) {
tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
if err != nil {
return
}
defer os.RemoveAll(tmp)

archive := tmp + "/" + file + ".tar.gz"
url := imagesBaseURL + file + ".tar.gz"

err = downloadFile(archive, url)
if err != nil {
return
}

err = unpackTar(archive, path)
return
}

+ 48
- 20
kernel.go View File

@@ -259,13 +259,24 @@ func genInitrdPath(files []os.FileInfo, kname string) string {
return "unknown"
}

func genRootfsImage(d dockerImageInfo) string {
func genRootfsImage(d dockerImageInfo, download bool) (rootfs string, err error) {
usr, err := user.Current()
if err != nil {
return fmt.Sprintln(err)
return
}
imageFile := d.ContainerName + ".img"
return usr.HomeDir + "/.out-of-tree/images/" + imageFile

imagesPath := usr.HomeDir + "/.out-of-tree/images/"
os.MkdirAll(imagesPath, os.ModePerm)

rootfs = imagesPath + imageFile
if !exists(rootfs) {
if download {
log.Println(imageFile, "not exists, start downloading...")
err = downloadImage(imagesPath, imageFile)
}
}
return
}

type dockerImageInfo struct {
@@ -309,7 +320,7 @@ func listDockerImages() (diis []dockerImageInfo, err error) {
return
}

func genHostKernels() (kcfg config.KernelConfig, err error) {
func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
si := sysinfo.SysInfo{}
si.GetSysInfo()

@@ -339,6 +350,11 @@ func genHostKernels() (kcfg config.KernelConfig, err error) {
}.DockerName(),
}

rootfs, err := genRootfsImage(dii, download)
if err != nil {
return
}

for _, k := range strings.Fields(string(rawOutput)) {
ki := config.KernelInfo{
DistroType: distroType,
@@ -349,7 +365,7 @@ func genHostKernels() (kcfg config.KernelConfig, err error) {

KernelPath: kernelsBase + genKernelPath(files, k),
InitrdPath: kernelsBase + genInitrdPath(files, k),
RootFS: genRootfsImage(dii),
RootFS: rootfs,
}

vmlinux := "/usr/lib/debug/boot/vmlinux-" + k
@@ -364,12 +380,12 @@ func genHostKernels() (kcfg config.KernelConfig, err error) {
return
}

func updateKernelsCfg(host bool) (err error) {
func updateKernelsCfg(host, download bool) (err error) {
newkcfg := config.KernelConfig{}

if host {
// Get host kernels
newkcfg, err = genHostKernels()
newkcfg, err = genHostKernels(download)
if err != nil {
return
}
@@ -382,7 +398,7 @@ func updateKernelsCfg(host bool) (err error) {
}

for _, d := range dockerImages {
err = genDockerKernels(d, &newkcfg)
err = genDockerKernels(d, &newkcfg, download)
if err != nil {
log.Println("gen kernels", d.ContainerName, ":", err)
continue
@@ -419,8 +435,8 @@ func updateKernelsCfg(host bool) (err error) {
return
}

func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) (
err error) {
func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig,
download bool) (err error) {

name := dii.ContainerName
cmd := exec.Command("docker", "run", name, "ls", "/lib/modules")
@@ -440,6 +456,11 @@ func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) (
return
}

rootfs, err := genRootfsImage(dii, download)
if err != nil {
return
}

for _, k := range strings.Fields(string(rawOutput)) {
ki := config.KernelInfo{
DistroType: dii.DistroType,
@@ -449,7 +470,7 @@ func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) (

KernelPath: kernelsBase + genKernelPath(files, k),
InitrdPath: kernelsBase + genInitrdPath(files, k),
RootFS: genRootfsImage(dii),
RootFS: rootfs,
}
newkcfg.Kernels = append(newkcfg.Kernels, ki)
}
@@ -475,8 +496,15 @@ func shuffle(a []string) []string {
return a
}

func generateKernels(km config.KernelMask, max int64) (err error) {
func generateKernels(km config.KernelMask, max int64, download bool) (err error) {
log.Println("Generating for kernel mask", km)

_, err = genRootfsImage(dockerImageInfo{ContainerName: km.DockerName()},
download)
if err != nil {
return
}

err = generateBaseDockerImage(km)
if err != nil {
return
@@ -519,7 +547,7 @@ func generateKernels(km config.KernelMask, max int64) (err error) {
return
}

func kernelAutogenHandler(workPath string, max int64, host bool) (err error) {
func kernelAutogenHandler(workPath string, max int64, host, download bool) (err error) {
ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
if err != nil {
return
@@ -531,17 +559,17 @@ func kernelAutogenHandler(workPath string, max int64, host bool) (err error) {
return
}

err = generateKernels(sk, max)
err = generateKernels(sk, max, download)
if err != nil {
return
}
}

err = updateKernelsCfg(host)
err = updateKernelsCfg(host, download)
return
}

func kernelDockerRegenHandler(host bool) (err error) {
func kernelDockerRegenHandler(host, download bool) (err error) {
dockerImages, err := listDockerImages()
if err != nil {
return
@@ -579,10 +607,10 @@ func kernelDockerRegenHandler(host bool) (err error) {
}
}

return updateKernelsCfg(host)
return updateKernelsCfg(host, download)
}

func kernelGenallHandler(distro, version string, host bool) (err error) {
func kernelGenallHandler(distro, version string, host, download bool) (err error) {
distroType, err := config.NewDistroType(distro)
if err != nil {
return
@@ -593,10 +621,10 @@ func kernelGenallHandler(distro, version string, host bool) (err error) {
DistroRelease: version,
ReleaseMask: ".*",
}
err = generateKernels(km, kernelsAll)
err = generateKernels(km, kernelsAll, download)
if err != nil {
return
}

return updateKernelsCfg(host)
return updateKernelsCfg(host, download)
}

+ 15
- 13
main.go View File

@@ -145,6 +145,8 @@ func main() {
pewTag := pewTagFlag.String()

kernelCommand := app.Command("kernel", "Manipulate kernels")
kernelNoDownload := kernelCommand.Flag("no-download",
"Do not download qemu image while kernel generation").Bool()
kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool()
kernelListCommand := kernelCommand.Command("list", "List kernels")
kernelAutogenCommand := kernelCommand.Command("autogen",
@@ -187,8 +189,7 @@ func main() {
nosmap := debugCommand.Flag("disable-smap", "Disable SMAP").Bool()
nokpti := debugCommand.Flag("disable-kpti", "Disable KPTI").Bool()

bootstrapCommand := app.Command("bootstrap",
"Create directories && download images")
bootstrapCommand := app.Command("bootstrap", "Apparently nothing")

logCommand := app.Command("log", "Logs")

@@ -209,6 +210,8 @@ func main() {

packCommand := app.Command("pack", "Exploit pack test")
packAutogen := packCommand.Flag("autogen", "Kernel autogeneration").Bool()
packNoDownload := packCommand.Flag("no-download",
"Do not download qemu image while kernel generation").Bool()
packExploitRuns := packCommand.Flag("exploit-runs",
"Amount of runs of each exploit").Default("4").Int64()
packKernelRuns := packCommand.Flag("kernel-runs",
@@ -228,12 +231,6 @@ func main() {
os.Exit(1)
}

if !exists(usr.HomeDir + "/.out-of-tree/images") {
log.Println("No ~/.out-of-tree/images: " +
"Probably you need to run `out-of-tree bootstrap`" +
" for downloading basic images")
}

if !exists(usr.HomeDir + "/.out-of-tree/kernels.toml") {
log.Println("No ~/.out-of-tree/kernels.toml: Probably you " +
"need to run `out-of-tree kernel autogen` in " +
@@ -290,11 +287,13 @@ func main() {
case kernelListCommand.FullCommand():
err = kernelListHandler(kcfg)
case kernelAutogenCommand.FullCommand():
err = kernelAutogenHandler(*path, *kernelAutogenMax, *kernelUseHost)
err = kernelAutogenHandler(*path, *kernelAutogenMax,
*kernelUseHost, !*kernelNoDownload)
case kernelDockerRegenCommand.FullCommand():
err = kernelDockerRegenHandler(*kernelUseHost)
err = kernelDockerRegenHandler(*kernelUseHost, !*kernelNoDownload)
case kernelGenallCommand.FullCommand():
err = kernelGenallHandler(*distro, *version, *kernelUseHost)
err = kernelGenallHandler(*distro, *version,
*kernelUseHost, !*kernelNoDownload)
case genModuleCommand.FullCommand():
err = genConfig(config.KernelModule)
case genExploitCommand.FullCommand():
@@ -304,7 +303,10 @@ func main() {
*dockerTimeout, *yekaslr, *yesmep, *yesmap, *yekpti,
*nokaslr, *nosmep, *nosmap, *nokpti)
case bootstrapCommand.FullCommand():
err = bootstrapHandler()
fmt.Println("bootstrap is no more required, " +
"now images downloading on-demand")
fmt.Println("please, remove it from any automation scripts, " +
"because it'll be removed in the next release")
case logQueryCommand.FullCommand():
err = logHandler(db, *path, *logTag, *logNum, *logRate)
case logDumpCommand.FullCommand():
@@ -315,7 +317,7 @@ func main() {
err = logMarkdownHandler(db, *path, *logMarkdownTag)
case packCommand.FullCommand():
err = packHandler(db, *path, kcfg, *packAutogen,
*packExploitRuns, *packKernelRuns)
!*packNoDownload, *packExploitRuns, *packKernelRuns)
}

if err != nil {

+ 2
- 2
pack.go View File

@@ -16,7 +16,7 @@ import (
)

func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,
autogen bool, exploitRuns, kernelRuns int64) (err error) {
autogen, download bool, exploitRuns, kernelRuns int64) (err error) {

dockerTimeout := time.Minute
qemuTimeout := time.Minute
@@ -39,7 +39,7 @@ func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,

if autogen {
var perRegex int64 = 1
err = kernelAutogenHandler(workPath, perRegex, false)
err = kernelAutogenHandler(workPath, perRegex, false, download)
if err != nil {
return
}

Loading…
Cancel
Save