tools
/
out-of-tree
Watch 1
Star 0
Fork 0
Code Releases 10 Activity
Browse Source

Refactor [2]

tags/v1.0.0
dump_stack() 1 year ago
parent
e0c0d3a072
commit
986a6f55e0
Signed by: Mikhail Klementev <blame@dumpstack.io> GPG Key ID: BE44DA8C062D87DC
8 changed files with 73 additions and 88 deletions
Split View Show Diff Stats
  1. 0
    16
      CHANGELOG.md
  2. 11
    11
      db.go
  3. 5
    5
      debug.go
  4. 1
    1
      main.go
  5. 7
    7
      pew.go
  6. 1
    1
      qemu/README.md
  7. 26
    25
      qemu/qemu-kernel.go
  8. 22
    22
      qemu/qemu-kernel_test.go

+ 0
- 16
CHANGELOG.md View File

@@ -12,33 +12,22 @@
base on `.out-of-tree.toml` definitions) and `pew` (automated runs)
and allows to specify a maximum number of runs per each supported
kernel in module/exploit definition.

- New command `genall` -- generate all kernels for specified
distro/version.

- All logs stores in sqlite3 database. Implemented specific commands
for making simple queries and export data to markdown and json.

- Implemented success rate calculation for previous runs.

- Save of build results supported by parameter `--dist` for `pew`.

- Support for generating kernels info from host system.

- Support for build on host.

- Support for custom kernels.

- Now debugging environment is automatically looking for debug kernel
on the host system.

- Added ability to enable/disable kaslr/smep/smap for debugging by
command line flags.

- New parameter `--threads=N` is added for `pew` and allows to specify
maximum number of threads that will be used for parallel
build/run/test.

- Tagging for runs. Tags write to log and can be used for statistics.

### Changed
@@ -46,13 +35,10 @@
- Now if there's no base image found — out-of-tree will try to use an
image from closest previous version, e.g. image from Ubuntu 18.04
for Ubuntu 18.10.

- Kernel modules tests will not be failed if there are no tests
exists.

- Now *out-of-tree* will return negative error code if at least one of
the stage was failed.

- Project is switch to use Go modules.

### Removed
@@ -63,9 +49,7 @@
### Fixed

- Command `timeout` is not required anymore.

- Errors is more meaningful.

- Temporary files is moved to `~/.out-of-tree/tmp/` to avoid docker mounting issues on some systems.

## [0.2.0] - 2019-12-01

+ 11
- 11
db.go View File

@@ -17,16 +17,16 @@ import (
)

// Change on ANY database update
const CURRENT_DATABASE_VERSION = 2
const currentDatabaseVersion = 2

const VERSION_FIELD = "db_version"
const versionField = "db_version"

type logEntry struct {
ID int
Tag string
Timestamp time.Time

qemu.QemuSystem
qemu.System
config.Artifact
config.KernelInfo
phasesResult
@@ -111,7 +111,7 @@ func metaSetValue(db *sql.DB, key, value string) (err error) {
}

func getVersion(db *sql.DB) (version int, err error) {
s, err := metaGetValue(db, VERSION_FIELD)
s, err := metaGetValue(db, versionField)
if err != nil {
return
}
@@ -120,7 +120,7 @@ func getVersion(db *sql.DB) (version int, err error) {
return
}

func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
func addToLog(db *sql.DB, q *qemu.System, ka config.Artifact,
ki config.KernelInfo, res *phasesResult, tag string) (err error) {

stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " +
@@ -276,15 +276,15 @@ func openDatabase(path string) (db *sql.DB, err error) {

db.SetMaxOpenConns(1)

exists, _ := metaChkValue(db, VERSION_FIELD)
exists, _ := metaChkValue(db, versionField)
if !exists {
err = createSchema(db)
if err != nil {
return
}

err = metaSetValue(db, VERSION_FIELD,
strconv.Itoa(CURRENT_DATABASE_VERSION))
err = metaSetValue(db, versionField,
strconv.Itoa(currentDatabaseVersion))
return
}

@@ -299,7 +299,7 @@ func openDatabase(path string) (db *sql.DB, err error) {
return
}

err = metaSetValue(db, VERSION_FIELD, "2")
err = metaSetValue(db, versionField, "2")
if err != nil {
return
}
@@ -307,9 +307,9 @@ func openDatabase(path string) (db *sql.DB, err error) {
version = 2
}

if version != CURRENT_DATABASE_VERSION {
if version != currentDatabaseVersion {
err = fmt.Errorf("Database is not supported (%d instead of %d)",
version, CURRENT_DATABASE_VERSION)
version, currentDatabaseVersion)
return
}


+ 5
- 5
debug.go View File

@@ -41,7 +41,7 @@ func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
return
}

func handleLine(q *qemu.QemuSystem) (err error) {
func handleLine(q *qemu.System) (err error) {
fmt.Print("out-of-tree> ")
rawLine := "help"
fmt.Scanf("%s", &rawLine)
@@ -64,7 +64,7 @@ func handleLine(q *qemu.QemuSystem) (err error) {
case "c", "cleanup":
q.Stdout = []byte{}
case "s", "ssh":
fmt.Println(q.GetSshCommand())
fmt.Println(q.GetSSHCommand())
case "q", "quit":
return errors.New("end of session")
default:
@@ -73,7 +73,7 @@ func handleLine(q *qemu.QemuSystem) (err error) {
return
}

func interactive(q *qemu.QemuSystem) (err error) {
func interactive(q *qemu.System) (err error) {
for {
err = handleLine(q)
if err != nil {
@@ -100,7 +100,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
}

kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS)
q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
if err != nil {
return
}
@@ -155,7 +155,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)

fmt.Printf("\n%s\n", q.GetSshCommand())
fmt.Printf("\n%s\n", q.GetSSHCommand())
fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, gdb)

// TODO set substitute-path /build/.../linux-... /path/to/linux-source

+ 1
- 1
main.go View File

@@ -136,7 +136,7 @@ func main() {
pewTest := pewTestFlag.String()

pewDistFlag := pewCommand.Flag("dist", "Build result path")
pewDist := pewDistFlag.Default(PATH_DEV_NULL).String()
pewDist := pewDistFlag.Default(pathDevNull).String()

pewThreadsFlag := pewCommand.Flag("threads", "Build result path")
pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int()

+ 7
- 7
pew.go View File

@@ -28,7 +28,7 @@ import (

var somethingFailed = false

const PATH_DEV_NULL = "/dev/null"
const pathDevNull = "/dev/null"

func dockerRun(timeout time.Duration, container, workdir, command string) (
output string, err error) {
@@ -101,7 +101,7 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
return
}

func cleanDmesg(q *qemu.QemuSystem) (err error) {
func cleanDmesg(q *qemu.System) (err error) {
start := time.Now()
for {
_, err = q.Command("root", "dmesg -c")
@@ -118,7 +118,7 @@ func cleanDmesg(q *qemu.QemuSystem) (err error) {
return
}

func testKernelModule(q *qemu.QemuSystem, ka config.Artifact,
func testKernelModule(q *qemu.System, ka config.Artifact,
test string) (output string, err error) {

output, err = q.Command("root", test)
@@ -126,7 +126,7 @@ func testKernelModule(q *qemu.QemuSystem, ka config.Artifact,
return
}

func testKernelExploit(q *qemu.QemuSystem, ka config.Artifact,
func testKernelExploit(q *qemu.System, ka config.Artifact,
test, exploit string) (output string, err error) {

output, err = q.Command("user", "chmod +x "+exploit)
@@ -188,7 +188,7 @@ func copyFile(sourcePath, destinationPath string) (err error) {
return destinationFile.Close()
}

func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
res *phasesResult, dist, tag, binary string, db *sql.DB) {

// TODO merge (problem is it's not 100% same) with log.go:logLogEntry
@@ -226,7 +226,7 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
log.Println("[db] addToLog (", ka, ") error:", err)
}

if binary == "" && dist != PATH_DEV_NULL {
if binary == "" && dist != pathDevNull {
err = os.MkdirAll(dist, os.ModePerm)
if err != nil {
log.Println("os.MkdirAll (", ka, ") error:", err)
@@ -253,7 +253,7 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
defer swg.Done()

kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS)
q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
if err != nil {
log.Println("Qemu creation error:", err)
return

+ 1
- 1
qemu/README.md View File

@@ -52,7 +52,7 @@ Minimal example:
KernelPath: "/path/to/vmlinuz",
InitrdPath: "/path/to/initrd", // if required
}
q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, "/path/to/qcow2")
q, err := qemu.NewSystem(qemu.X86_64, kernel, "/path/to/qcow2")
if err != nil {
log.Fatal(err)
}

+ 26
- 25
qemu/qemu-kernel.go View File

@@ -44,9 +44,10 @@ func readUntilEOF(pipe io.ReadCloser, buf *[]byte) (err error) {
type arch string

const (
// X86_64 must be exactly same as in qemu-system-${HERE}
X86_64 arch = "x86_64"
I386 = "i386"
// X86x64 is the qemu-system-x86_64
X86x64 arch = "x86_64"
// X86x32 is the qemu-system-i386
X86x32 = "i386"
// TODO add other

unsupported = "unsupported" // for test purposes
@@ -59,8 +60,8 @@ type Kernel struct {
InitrdPath string
}

// QemuSystem describe qemu parameters and runned process
type QemuSystem struct {
// System describe qemu parameters and runned process
type System struct {
arch arch
kernel Kernel
drivePath string
@@ -98,12 +99,12 @@ type QemuSystem struct {
exitErr error
}

// NewQemuSystem constructor
func NewQemuSystem(arch arch, kernel Kernel, drivePath string) (q *QemuSystem, err error) {
// NewSystem constructor
func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error) {
if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil {
return
}
q = &QemuSystem{}
q = &System{}
q.arch = arch

if _, err = os.Stat(kernel.KernelPath); err != nil {
@@ -164,7 +165,7 @@ func kvmExists() bool {
return true
}

func (q *QemuSystem) panicWatcher() {
func (q *System) panicWatcher() {
for {
time.Sleep(time.Second)
if bytes.Contains(q.Stdout, []byte("Kernel panic")) {
@@ -178,7 +179,7 @@ func (q *QemuSystem) panicWatcher() {
}

// Start qemu process
func (q *QemuSystem) Start() (err error) {
func (q *System) Start() (err error) {
rand.Seed(time.Now().UnixNano()) // Are you sure?
q.sshAddrPort = getFreeAddrPort()
hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.sshAddrPort)
@@ -213,11 +214,11 @@ func (q *QemuSystem) Start() (err error) {
qemuArgs = append(qemuArgs, "-initrd", q.kernel.InitrdPath)
}

if (q.arch == X86_64 || q.arch == I386) && kvmExists() {
if (q.arch == X86x64 || q.arch == X86x32) && kvmExists() {
qemuArgs = append(qemuArgs, "-enable-kvm")
}

if q.arch == X86_64 && runtime.GOOS == "darwin" {
if q.arch == X86x64 && runtime.GOOS == "darwin" {
qemuArgs = append(qemuArgs, "-accel", "hvf", "-cpu", "host")
}

@@ -270,7 +271,7 @@ func (q *QemuSystem) Start() (err error) {
}

// Stop qemu process
func (q *QemuSystem) Stop() {
func (q *System) Stop() {
// 1 00/01 01 01 SOH (Ctrl-A) START OF HEADING
fmt.Fprintf(q.pipe.stdin, "%cx", 1)
// wait for die
@@ -282,7 +283,7 @@ func (q *QemuSystem) Stop() {
}
}

func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) {
func (q System) ssh(user string) (client *ssh.Client, err error) {
cfg := &ssh.ClientConfig{
User: user,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
@@ -293,7 +294,7 @@ func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) {
}

// Command executes shell commands on qemu system
func (q QemuSystem) Command(user, cmd string) (output string, err error) {
func (q System) Command(user, cmd string) (output string, err error) {
client, err := q.ssh(user)
if err != nil {
return
@@ -311,7 +312,7 @@ func (q QemuSystem) Command(user, cmd string) (output string, err error) {
}

// AsyncCommand executes command on qemu system but does not wait for exit
func (q QemuSystem) AsyncCommand(user, cmd string) (err error) {
func (q System) AsyncCommand(user, cmd string) (err error) {
client, err := q.ssh(user)
if err != nil {
return
@@ -328,7 +329,7 @@ func (q QemuSystem) AsyncCommand(user, cmd string) (err error) {
}

// CopyFile is copy file from local machine to remote through ssh/scp
func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) {
func (q *System) CopyFile(user, localPath, remotePath string) (err error) {
addrPort := strings.Split(q.sshAddrPort, ":")
addr := addrPort[0]
port := addrPort[1]
@@ -346,7 +347,7 @@ func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) {
}

// CopyAndInsmod copy kernel module to temporary file on qemu then insmod it
func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error) {
func (q *System) CopyAndInsmod(localKoPath string) (output string, err error) {
remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int())
err = q.CopyFile("root", localKoPath, remoteKoPath)
if err != nil {
@@ -357,7 +358,7 @@ func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error
}

// CopyAndRun is copy local file to qemu vm then run it
func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) {
func (q *System) CopyAndRun(user, path string) (output string, err error) {
remotePath := fmt.Sprintf("/tmp/executable_%d", rand.Int())
err = q.CopyFile(user, path, remotePath)
if err != nil {
@@ -368,28 +369,28 @@ func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) {
}

// Debug is for enable qemu debug and set hostname and port for listen
func (q *QemuSystem) Debug(conn string) {
func (q *System) Debug(conn string) {
q.debug = true
q.gdb = conn
}

// SetKASLR is changing KASLR state through kernel boot args
func (q *QemuSystem) SetKASLR(state bool) {
func (q *System) SetKASLR(state bool) {
q.noKASLR = !state
}

// SetSMEP is changing SMEP state through kernel boot args
func (q *QemuSystem) SetSMEP(state bool) {
func (q *System) SetSMEP(state bool) {
q.noSMEP = !state
}

// SetSMAP is changing SMAP state through kernel boot args
func (q *QemuSystem) SetSMAP(state bool) {
func (q *System) SetSMAP(state bool) {
q.noSMAP = !state
}

// GetSshCommand returns command for connect to qemu machine over ssh
func (q QemuSystem) GetSshCommand() (cmd string) {
// GetSSHCommand returns command for connect to qemu machine over ssh
func (q System) GetSSHCommand() (cmd string) {
addrPort := strings.Split(q.sshAddrPort, ":")
addr := addrPort[0]
port := addrPort[1]

+ 22
- 22
qemu/qemu-kernel_test.go View File

@@ -20,37 +20,37 @@ func init() {
rand.Seed(time.Now().UnixNano())
}

func TestQemuSystemNew_InvalidKernelPath(t *testing.T) {
func TestSystemNew_InvalidKernelPath(t *testing.T) {
kernel := Kernel{Name: "Invalid", KernelPath: "/invalid/path"}
if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err == nil {
if _, err := NewSystem(X86_64, kernel, "/bin/sh"); err == nil {
t.Fatal(err)
}
}

func TestQemuSystemNew_InvalidQemuArch(t *testing.T) {
func TestSystemNew_InvalidQemuArch(t *testing.T) {
kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
if _, err := NewQemuSystem(unsupported, kernel, "/bin/sh"); err == nil {
if _, err := NewSystem(unsupported, kernel, "/bin/sh"); err == nil {
t.Fatal(err)
}
}

func TestQemuSystemNew_InvalidQemuDrivePath(t *testing.T) {
func TestSystemNew_InvalidQemuDrivePath(t *testing.T) {
kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
if _, err := NewQemuSystem(X86_64, kernel, "/invalid/path"); err == nil {
if _, err := NewSystem(X86_64, kernel, "/invalid/path"); err == nil {
t.Fatal(err)
}
}

func TestQemuSystemNew(t *testing.T) {
func TestSystemNew(t *testing.T) {
kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err != nil {
if _, err := NewSystem(X86_64, kernel, "/bin/sh"); err != nil {
t.Fatal(err)
}
}

func TestQemuSystemStart(t *testing.T) {
func TestSystemStart(t *testing.T) {
kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
q, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
q, err := NewSystem(X86_64, kernel, "/bin/sh")
if err != nil {
t.Fatal(err)
}
@@ -71,10 +71,10 @@ func TestGetFreeAddrPort(t *testing.T) {
ln.Close()
}

func TestQemuSystemStart_Timeout(t *testing.T) {
func TestSystemStart_Timeout(t *testing.T) {
t.Parallel()
kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
q, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
q, err := NewSystem(X86_64, kernel, "/bin/sh")
if err != nil {
t.Fatal(err)
}
@@ -96,14 +96,14 @@ func TestQemuSystemStart_Timeout(t *testing.T) {
}
}

func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err error) {
func startTestQemu(t *testing.T, timeout time.Duration) (q *System, err error) {
t.Parallel()
kernel := Kernel{
Name: "Test kernel",
KernelPath: testConfigVmlinuz,
InitrdPath: testConfigInitrd,
}
q, err = NewQemuSystem(X86_64, kernel, testConfigRootfs)
q, err = NewSystem(X86_64, kernel, testConfigRootfs)
if err != nil {
return
}
@@ -119,7 +119,7 @@ func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err erro
return
}

func TestQemuSystemCommand(t *testing.T) {
func TestSystemCommand(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@@ -149,7 +149,7 @@ func TestQemuSystemCommand(t *testing.T) {
}
}

func TestQemuSystemCopyFile(t *testing.T) {
func TestSystemCopyFile(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@@ -182,7 +182,7 @@ func TestQemuSystemCopyFile(t *testing.T) {
}
}

func TestQemuSystemCopyAndRun(t *testing.T) {
func TestSystemCopyAndRun(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@@ -216,7 +216,7 @@ func TestQemuSystemCopyAndRun(t *testing.T) {
}
}

func TestQemuSystemCopyAndInsmod(t *testing.T) {
func TestSystemCopyAndInsmod(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@@ -243,7 +243,7 @@ func TestQemuSystemCopyAndInsmod(t *testing.T) {
}
}

func TestQemuSystemKernelPanic(t *testing.T) {
func TestSystemKernelPanic(t *testing.T) {
q, err := startTestQemu(t, 5*time.Minute)
if err != nil {
t.Fatal(err)
@@ -278,7 +278,7 @@ func TestQemuSystemKernelPanic(t *testing.T) {
}
}

func TestQemuSystemRun(t *testing.T) {
func TestSystemRun(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@@ -313,13 +313,13 @@ func openedPort(port int) bool {
return true
}

func TestQemuSystemDebug(t *testing.T) {
func TestSystemDebug(t *testing.T) {
t.Parallel()
kernel := Kernel{
KernelPath: testConfigVmlinuz,
InitrdPath: testConfigInitrd,
}
q, err := NewQemuSystem(X86_64, kernel, testConfigRootfs)
q, err := NewSystem(X86_64, kernel, testConfigRootfs)
if err != nil {
return
}

Write Preview
Loading…
Cancel
Save