out-of-tree/distro/debian/debian.go

package debian

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/rs/zerolog/log"

	"code.dumpstack.io/tools/out-of-tree/cache"
	"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
	"code.dumpstack.io/tools/out-of-tree/container"
	"code.dumpstack.io/tools/out-of-tree/distro"
	"code.dumpstack.io/tools/out-of-tree/distro/debian/snapshot"
	"code.dumpstack.io/tools/out-of-tree/fs"
)

func init() {
	releases := []Release{
		Wheezy,
		Jessie,
		Stretch,
		Buster,
		Bullseye,
		Bookworm,
	}

	for _, release := range releases {
		distro.Register(Debian{release: release})
	}
}

type Debian struct {
	release Release
}

func (d Debian) Equal(dd distro.Distro) bool {
	if dd.ID != distro.Debian {
		return false
	}

	return ReleaseFromString(dd.Release) == d.release
}

func (d Debian) Distro() distro.Distro {
	return distro.Distro{distro.Debian, d.release.String()}
}

func (d Debian) Packages() (packages []string, err error) {
	c, err := container.New(d.Distro())
	if err != nil {
		return
	}

	if !c.Exist() {
		err = c.Build(d.image(), d.envs(), d.runs())
		if err != nil {
			return
		}
	}

	kernels, err := GetKernels()
	if err != nil {
		log.Error().Err(err).Msg("get kernels")
		return
	}

	for _, dk := range kernels {
		if d.release != dk.Release {
			continue
		}

		version := kver(dk.Version.Package)

		// filter out pre-release kernels
		switch dk.Release {
		case Wheezy:
			if version.LessThan(kver("3.2-rc0")) {
				continue
			}
		case Jessie:
			if version.LessThan(kver("3.16-rc0")) {
				continue
			}
		case Stretch:
			if version.LessThan(kver("4.9-rc0")) {
				continue
			}
		case Buster:
			if version.LessThan(kver("4.19-rc0")) {
				continue
			}
		case Bullseye:
			if version.LessThan(kver("5.10-rc0")) {
				continue
			}
		case Bookworm:
			if version.LessThan(kver("6.1-rc0")) {
				continue
			}
		}

		p := dk.Image.Deb.Name[:len(dk.Image.Deb.Name)-4] // w/o .deb
		packages = append(packages, p)
	}

	return
}

type Release int

const (
	None Release = iota
	Buzz
	Hamm
	Woody
	Etch
	Lenny
	Squeeze
	Wheezy
	Jessie
	Stretch
	Buster
	Bullseye
	Bookworm
)

var ReleaseStrings = [...]string{
	"",
	"buzz",
	"hamm",
	"woody",
	"etch",
	"lenny",
	"squeeze",
	"wheezy",
	"jessie",
	"stretch",
	"buster",
	"bullseye",
	"bookworm",
}

func (cn Release) Name() string {
	return ReleaseStrings[cn]
}

func (cn Release) String() string {
	return fmt.Sprintf("%d", cn)
}

func ReleaseFromString(s string) (r Release) {
	switch strings.ToLower(s) {
	case "1", "buzz":
		r = Buzz
	case "2", "hamm":
		r = Hamm
	case "3", "woody":
		r = Woody
	case "4", "etch":
		r = Etch
	case "5", "lenny":
		r = Lenny
	case "6", "squeeze":
		r = Squeeze
	case "7", "wheezy":
		r = Wheezy
	case "8", "jessie":
		r = Jessie
	case "9", "stretch":
		r = Stretch
	case "10", "buster":
		r = Buster
	case "11", "bullseye":
		r = Bullseye
	case "12", "bookworm":
		r = Bookworm
	default:
		r = None
	}
	return
}

func (d Debian) envs() (envs []string) {
	envs = append(envs, "DEBIAN_FRONTEND=noninteractive")
	return
}

func (d Debian) image() (image string) {
	image += "debian:"

	switch d.release {
	case Wheezy:
		image += "wheezy-20190228"
	case Jessie:
		image += "jessie-20210326"
	case Stretch:
		image += "stretch-20220622"
	default:
		image += d.release.Name()
	}

	return
}

func repositories(release Release) (repos []string) {
	var snapshot string

	switch release {
	// Latest snapshots that include release
	case Wheezy:
		// doesn't include snapshot repos in /etc/apt/source.list
		snapshot = "20190321T212815Z"
	case Jessie:
		snapshot = "20230322T152120Z"
	case Stretch:
		snapshot = "20230423T032533Z"
	default:
		return
	}

	repo := func(archive, s string) {
		format := "deb [check-valid-until=no trusted=yes] " +
			"http://snapshot.debian.org/archive/%s/%s " +
			"%s%s main"
		r := fmt.Sprintf(format, archive, snapshot, release.Name(), s)
		repos = append(repos, r)
	}

	repo("debian", "")
	repo("debian", "-updates")
	if release <= 7 {
		repo("debian", "-backports")
	}
	repo("debian-security", "/updates")

	return
}

func (d Debian) runs() (commands []string) {
	cmdf := func(f string, s ...interface{}) {
		commands = append(commands, fmt.Sprintf(f, s...))
	}

	repos := repositories(d.release)

	if len(repos) != 0 {
		cmdf("rm /etc/apt/sources.list")
		for _, repo := range repos {
			cmdf("echo '%s' >> /etc/apt/sources.list", repo)
		}
	} else {
		cmdf("apt-get update || sed -i " +
			"-e '/snapshot/!d' " +
			"-e 's/# deb/deb [check-valid-until=no trusted=yes]/' " +
			"/etc/apt/sources.list")
	}

	cmdf("apt-get update || apt-get update || apt-get update")

	pkglist := []string{
		"wget", "build-essential", "libelf-dev", "git",
		"kmod", "linux-base", "libssl-dev",
		"firmware-linux-free",
		"libxml2", "libglib2.0.0", "irqbalance", "libcap-ng0",
		"libnuma1", "sgml-base", "shared-mime-info", "xdg-user-dirs",
		"xml-core", "python3",
	}

	gccs := "'^(gcc-[0-9].[0-9]|gcc-[0-9]|gcc-[1-9][0-9])$'"
	pkglist = append(pkglist, gccs)

	if d.release >= 8 {
		pkglist = append(pkglist, "initramfs-tools")
	} else {
		// by default Debian backports repositories have a lower
		// priority than stable, so we should specify it manually
		cmdf("apt-get -y install -t %s-backports "+
			"initramfs-tools", d.release.Name())
	}

	if d.release >= 9 {
		pkglist = append(pkglist, "apparmor")
	}

	if d.release < 9 {
		pkglist = append(pkglist, "module-init-tools")
	}

	var packages string
	for _, pkg := range pkglist {
		packages += fmt.Sprintf("%s ", pkg)
	}

	cmdf("timeout 5m apt-get install -y %s "+
		"|| timeout 10m apt-get install -y %s "+
		"|| apt-get install -y %s", packages, packages, packages)

	if d.release == Wheezy {
		// We need newer libc for deb8*~bpo70+1
		format := "deb [check-valid-until=no trusted=yes] " +
			"http://snapshot.debian.org/archive/debian/%s " +
			"jessie main"
		// Keep it here not in repos to have apt-priority close
		repo := fmt.Sprintf(format, "20190321T212815Z")
		cmdf("echo '%s' >> /etc/apt/sources.list", repo)
		cmdf("echo 'Package: *' >> /etc/apt/preferences.d/jessie")
		cmdf("echo 'Pin: release a=jessie' >> /etc/apt/preferences.d/jessie")
		cmdf("echo 'Pin-Priority: 10' >> /etc/apt/preferences.d/jessie")

		cmdf("apt-get -y update")

		// glibc guarantee backwards compatibility, so should be no problem
		cmdf("apt-get -y install -t jessie libc6-dev")
	}

	cmdf("mkdir -p /lib/modules")

	return
}

func (d Debian) Kernels() (kernels []distro.KernelInfo, err error) {
	c, err := container.New(d.Distro())
	if err != nil {
		return
	}

	if !c.Exist() {
		return
	}

	cpath := dotfiles.Dir("volumes", c.Name())
	rootfs := dotfiles.File("images", c.Name()+".img")

	files, err := os.ReadDir(cpath)
	if err != nil {
		return
	}

	for _, file := range files {
		if !strings.Contains(file.Name(), "linux-image") {
			continue
		}

		pkgname := file.Name()

		kpkgdir := filepath.Join(cpath, pkgname)

		bootdir := filepath.Join(kpkgdir, "boot")

		vmlinuz, err := fs.FindBySubstring(bootdir, "vmlinuz")
		if err != nil {
			log.Warn().Msgf("cannot find vmlinuz for %s", pkgname)
			continue
		}

		initrd, err := fs.FindBySubstring(bootdir, "initrd")
		if err != nil {
			log.Warn().Msgf("cannot find initrd for %s", pkgname)
			continue
		}

		modulesdir := filepath.Join(kpkgdir, "lib/modules")

		modules, err := fs.FindBySubstring(modulesdir, "")
		if err != nil {
			log.Warn().Msgf("cannot find modules for %s", pkgname)
			continue
		}

		log.Debug().Msgf("%s %s %s", vmlinuz, initrd, modules)

		release := strings.Replace(pkgname, "linux-image-", "", -1)

		ki := distro.KernelInfo{
			Distro:        d.Distro(),
			KernelVersion: path.Base(modules),
			KernelRelease: release,
			ContainerName: c.Name(),

			KernelPath:  vmlinuz,
			InitrdPath:  initrd,
			ModulesPath: modules,

			RootFS: rootfs,

			Package: pkgname,
		}

		smapBlocklist := []string{
			"3.10.5-1~bpo70+1",
			"3.10.11-1~bpo70+1",
			"3.9.6-1~bpo70+1",
		}
		for _, ver := range smapBlocklist {
			if strings.Contains(release, ver) {
				ki.CPU.Flags = append(ki.CPU.Flags, "smap=off")
			}
		}

		kernels = append(kernels, ki)
	}

	return
}

func (d Debian) volumes(pkgname string) (volumes []container.Volume) {
	c, err := container.New(d.Distro())
	if err != nil {
		return
	}

	pkgdir := filepath.Join("volumes", c.Name(), pkgname)

	volumes = append(volumes, container.Volume{
		Src:  dotfiles.Dir(pkgdir, "/lib/modules"),
		Dest: "/lib/modules",
	})

	volumes = append(volumes, container.Volume{
		Src:  dotfiles.Dir(pkgdir, "/usr/src"),
		Dest: "/usr/src",
	})

	volumes = append(volumes, container.Volume{
		Src:  dotfiles.Dir(pkgdir, "/boot"),
		Dest: "/boot",
	})

	return
}

func (d Debian) Install(pkgname string, headers bool) (err error) {
	defer func() {
		if err != nil {
			d.cleanup(pkgname)
		}
	}()

	dk, err := getCachedKernel(pkgname + ".deb")
	if err != nil {
		return
	}

	var pkgs []snapshot.Package
	if headers {
		pkgs = dk.Packages()
	} else {
		pkgs = []snapshot.Package{dk.Image}
	}

	var commands []string
	cmdf := func(f string, s ...interface{}) {
		commands = append(commands, fmt.Sprintf(f, s...))
	}

	for _, pkg := range pkgs {
		found, newurl := cache.PackageURL(
			distro.Debian,
			pkg.Deb.URL,
		)
		if found {
			log.Debug().Msgf("cached deb found %s", newurl)
			pkg.Deb.URL = newurl
		}

		// TODO use faketime on old releases?
		pkg.Deb.URL = strings.Replace(pkg.Deb.URL, "https", "http", -1)

		cmdf("wget --no-verbose " +
			"--timeout=10 --waitretry=1 --tries=10 " +
			"--no-check-certificate " + pkg.Deb.URL)
	}

	// prepare local repository
	cmdf("mkdir debs && mv *.deb debs/")
	cmdf("dpkg-scanpackages debs /dev/null | gzip > debs/Packages.gz")
	cmdf(`echo "deb [trusted=yes] file:$(pwd) debs/" >> /etc/apt/sources.list.d/local.list`)
	cmdf("apt-get update -o Dir::Etc::sourcelist='sources.list.d/local.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0'")

	// make sure apt-get will not download the repo version
	cmdf("echo 'Package: *' >> /etc/apt/preferences.d/pin")
	cmdf(`echo 'Pin: origin "*.debian.org"' >> /etc/apt/preferences.d/pin`)
	cmdf("echo 'Pin-Priority: 100' >> /etc/apt/preferences.d/pin")

	// cut package names and install
	cmdf("ls debs | grep deb | cut -d '_' -f 1 | " +
		"xargs apt-get -y --force-yes install")

	// for debug
	cmdf("ls debs | grep deb | cut -d '_' -f 1 | xargs apt-cache policy")

	c, err := container.New(d.Distro())
	if err != nil {
		return
	}

	c.Volumes = d.volumes(pkgname)
	for i := range c.Volumes {
		c.Volumes[i].Dest = "/target" + c.Volumes[i].Dest
	}

	cmdf("cp -r /boot /target/")
	cmdf("cp -r /lib/modules /target/lib/")
	cmdf("cp -rL /usr/src /target/usr/")

	_, err = c.Run("", commands)
	if err != nil {
		return
	}

	return
}

func (d Debian) cleanup(pkgname string) {
	c, err := container.New(d.Distro())
	if err != nil {
		return
	}

	pkgdir := dotfiles.Dir(filepath.Join("volumes", c.Name(), pkgname))

	log.Debug().Msgf("cleanup %s", pkgdir)

	err = os.RemoveAll(pkgdir)
	if err != nil {
		log.Warn().Err(err).Msg("cleanup")
	}
}

func (d Debian) RootFS() string {
	return fmt.Sprintf("out_of_tree_debian_%s.img", d.release.String())
}