Compare commits
No commits in common. "f33ff2570878c3dc0397bdc24c5c8052d0fd58fc" and "75dd8f4a5118e5a184d8b860f4825c37657ffeb9" have entirely different histories.
f33ff25708
...
75dd8f4a51
2
.github/workflows/e2e.yml
vendored
2
.github/workflows/e2e.yml
vendored
@ -139,7 +139,7 @@ jobs:
|
|||||||
echo 'Type=oneshot' >> test.service
|
echo 'Type=oneshot' >> test.service
|
||||||
echo 'WorkingDirectory=/root/test' >> test.service
|
echo 'WorkingDirectory=/root/test' >> test.service
|
||||||
echo 'TimeoutStopSec=1' >> test.service
|
echo 'TimeoutStopSec=1' >> test.service
|
||||||
echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-prebuilt-containers autogen --threads=8 --max=64 --shuffle' >> test.service
|
echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-container-cache autogen --threads=8 --max=64 --shuffle' >> test.service
|
||||||
echo 'ExecStart=/usr/local/bin/out-of-tree pew --qemu-timeout=10m --threads=4 --include-internal-errors' >> test.service
|
echo 'ExecStart=/usr/local/bin/out-of-tree pew --qemu-timeout=10m --threads=4 --include-internal-errors' >> test.service
|
||||||
|
|
||||||
scp test.service root@$IP:/etc/systemd/system/test.service
|
scp test.service root@$IP:/etc/systemd/system/test.service
|
||||||
|
@ -11,31 +11,34 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/cavaliergopher/grab/v3"
|
||||||
"github.com/naoina/toml"
|
"github.com/naoina/toml"
|
||||||
"github.com/remeh/sizedwaitgroup"
|
"github.com/remeh/sizedwaitgroup"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
|
|
||||||
"code.dumpstack.io/tools/out-of-tree/artifact"
|
"code.dumpstack.io/tools/out-of-tree/artifact"
|
||||||
|
"code.dumpstack.io/tools/out-of-tree/cache"
|
||||||
"code.dumpstack.io/tools/out-of-tree/config"
|
"code.dumpstack.io/tools/out-of-tree/config"
|
||||||
"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
|
"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
|
||||||
"code.dumpstack.io/tools/out-of-tree/container"
|
"code.dumpstack.io/tools/out-of-tree/container"
|
||||||
"code.dumpstack.io/tools/out-of-tree/distro"
|
"code.dumpstack.io/tools/out-of-tree/distro"
|
||||||
|
"code.dumpstack.io/tools/out-of-tree/fs"
|
||||||
"code.dumpstack.io/tools/out-of-tree/kernel"
|
"code.dumpstack.io/tools/out-of-tree/kernel"
|
||||||
)
|
)
|
||||||
|
|
||||||
type KernelCmd struct {
|
type KernelCmd struct {
|
||||||
NoDownload bool `help:"do not download qemu image while kernel generation"`
|
NoDownload bool `help:"do not download qemu image while kernel generation"`
|
||||||
UseHost bool `help:"also use host kernels"`
|
UseHost bool `help:"also use host kernels"`
|
||||||
Force bool `help:"force reinstall kernel"`
|
Force bool `help:"force reinstall kernel"`
|
||||||
NoHeaders bool `help:"do not install kernel headers"`
|
NoHeaders bool `help:"do not install kernel headers"`
|
||||||
Shuffle bool `help:"randomize kernels installation order"`
|
Shuffle bool `help:"randomize kernels installation order"`
|
||||||
Retries int `help:"amount of tries for each kernel" default:"2"`
|
Retries int `help:"amount of tries for each kernel" default:"2"`
|
||||||
Threads int `help:"threads for parallel installation" default:"1"`
|
Threads int `help:"threads for parallel installation" default:"1"`
|
||||||
Update bool `help:"update container"`
|
Update bool `help:"update container"`
|
||||||
PrebuiltContainers bool `help:"try prebuilt container images first" default:"true" negatable:""`
|
ContainerCache bool `help:"try prebuilt container images first" default:"true" negatable:""`
|
||||||
Max int `help:"maximum kernels to download" default:"100500"`
|
Max int `help:"maximum kernels to download" default:"100500"`
|
||||||
NoPrune bool `help:"do not remove dangling or unused images from local storage after build"`
|
NoPrune bool `help:"do not remove dangling or unused images from local storage after build"`
|
||||||
NoCfgRegen bool `help:"do not update kernels.toml"`
|
NoCfgRegen bool `help:"do not update kernels.toml"`
|
||||||
|
|
||||||
ContainerTimeout time.Duration `help:"container timeout"`
|
ContainerTimeout time.Duration `help:"container timeout"`
|
||||||
|
|
||||||
@ -165,6 +168,33 @@ func (cmd *KernelCmd) GenKernel(km artifact.Target, pkg string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cmd *KernelCmd) fetchContainerCache(c container.Container) {
|
||||||
|
if !cmd.ContainerCache {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c.Exist() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
tmp, err := fs.TempDir()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmp)
|
||||||
|
|
||||||
|
resp, err := grab.Get(tmp, cache.ContainerURL(c.Name()))
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer os.Remove(resp.Filename)
|
||||||
|
|
||||||
|
err = container.Load(resp.Filename, c.Name())
|
||||||
|
if err == nil {
|
||||||
|
log.Info().Msgf("use prebuilt container %s", c.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
|
func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
|
||||||
defer func() {
|
defer func() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -181,8 +211,6 @@ func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
|
|||||||
container.Prune = false
|
container.Prune = false
|
||||||
}
|
}
|
||||||
|
|
||||||
container.UsePrebuilt = cmd.PrebuiltContainers
|
|
||||||
|
|
||||||
cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
|
cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Err(err).Msg("read kernels config")
|
log.Debug().Err(err).Msg("read kernels config")
|
||||||
@ -202,6 +230,13 @@ func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
c, err := container.New(km.Distro)
|
||||||
|
if err != nil || cmd.shutdown {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.fetchContainerCache(c)
|
||||||
|
|
||||||
pkgs, err := kernel.MatchPackages(km)
|
pkgs, err := kernel.MatchPackages(km)
|
||||||
if err != nil || cmd.shutdown {
|
if err != nil || cmd.shutdown {
|
||||||
return
|
return
|
||||||
@ -277,8 +312,6 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
|
|||||||
container.Prune = false
|
container.Prune = false
|
||||||
}
|
}
|
||||||
|
|
||||||
container.UsePrebuilt = kernelCmd.PrebuiltContainers
|
|
||||||
|
|
||||||
distroType, err := distro.NewID(cmd.Distro)
|
distroType, err := distro.NewID(cmd.Distro)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
@ -297,6 +330,13 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
|
|||||||
container.Registry = g.Config.Docker.Registry
|
container.Registry = g.Config.Docker.Registry
|
||||||
container.Commands = g.Config.Docker.Commands
|
container.Commands = g.Config.Docker.Commands
|
||||||
|
|
||||||
|
c, err := container.New(km.Distro)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
kernelCmd.fetchContainerCache(c)
|
||||||
|
|
||||||
pkgs, err := kernel.MatchPackages(km)
|
pkgs, err := kernel.MatchPackages(km)
|
||||||
// error check skipped on purpose
|
// error check skipped on purpose
|
||||||
|
|
||||||
|
@ -16,14 +16,11 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/cavaliergopher/grab/v3"
|
|
||||||
"github.com/rs/zerolog"
|
"github.com/rs/zerolog"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
|
|
||||||
"code.dumpstack.io/tools/out-of-tree/cache"
|
|
||||||
"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
|
"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
|
||||||
"code.dumpstack.io/tools/out-of-tree/distro"
|
"code.dumpstack.io/tools/out-of-tree/distro"
|
||||||
"code.dumpstack.io/tools/out-of-tree/fs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var Runtime = "docker"
|
var Runtime = "docker"
|
||||||
@ -36,8 +33,6 @@ var Commands []distro.Command
|
|||||||
|
|
||||||
var UseCache = true
|
var UseCache = true
|
||||||
|
|
||||||
var UsePrebuilt = true
|
|
||||||
|
|
||||||
var Prune = true
|
var Prune = true
|
||||||
|
|
||||||
type Image struct {
|
type Image struct {
|
||||||
@ -245,38 +240,7 @@ func (c Container) Exist() (yes bool) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c Container) loadPrebuilt() (err error) {
|
|
||||||
if c.Exist() && UseCache {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
tmp, err := fs.TempDir()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer os.RemoveAll(tmp)
|
|
||||||
|
|
||||||
log.Info().Msgf("download prebuilt container %s", c.Name())
|
|
||||||
resp, err := grab.Get(tmp, cache.ContainerURL(c.Name()))
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
defer os.Remove(resp.Filename)
|
|
||||||
|
|
||||||
err = Load(resp.Filename, c.Name())
|
|
||||||
if err == nil {
|
|
||||||
log.Info().Msgf("use prebuilt container %s", c.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c Container) Build(image string, envs, runs []string) (err error) {
|
func (c Container) Build(image string, envs, runs []string) (err error) {
|
||||||
if c.Exist() && UseCache {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cdir := dotfiles.Dir("containers", c.name)
|
cdir := dotfiles.Dir("containers", c.name)
|
||||||
cfile := filepath.Join(cdir, "Dockerfile")
|
cfile := filepath.Join(cdir, "Dockerfile")
|
||||||
|
|
||||||
@ -322,17 +286,10 @@ func (c Container) Build(image string, envs, runs []string) (err error) {
|
|||||||
c.Log.Info().Msg("build")
|
c.Log.Info().Msg("build")
|
||||||
}
|
}
|
||||||
|
|
||||||
if UsePrebuilt {
|
output, err := c.build(cdir)
|
||||||
err = c.loadPrebuilt()
|
if err != nil {
|
||||||
}
|
c.Log.Error().Err(err).Msg(output)
|
||||||
|
return
|
||||||
if err != nil || !UsePrebuilt {
|
|
||||||
var output string
|
|
||||||
output, err = c.build(cdir)
|
|
||||||
if err != nil {
|
|
||||||
c.Log.Error().Err(err).Msg(output)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Log.Info().Msg("success")
|
c.Log.Info().Msg("success")
|
||||||
|
@ -37,9 +37,12 @@ func (centos CentOS) Packages() (pkgs []string, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = c.Build("centos:"+centos.release, centos.envs(), centos.runs())
|
if !c.Exist() {
|
||||||
if err != nil {
|
err = c.Build("centos:"+centos.release,
|
||||||
return
|
centos.envs(), centos.runs())
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd := "yum search kernel --showduplicates 2>/dev/null " +
|
cmd := "yum search kernel --showduplicates 2>/dev/null " +
|
||||||
|
@ -54,9 +54,11 @@ func (d Debian) Packages() (packages []string, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = c.Build(d.image(), d.envs(), d.runs())
|
if !c.Exist() {
|
||||||
if err != nil {
|
err = c.Build(d.image(), d.envs(), d.runs())
|
||||||
return
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
kernels, err := GetKernels()
|
kernels, err := GetKernels()
|
||||||
|
@ -70,9 +70,11 @@ func (suse OpenSUSE) Packages() (pkgs []string, err error) {
|
|||||||
name = "opensuse/leap:" + suse.release
|
name = "opensuse/leap:" + suse.release
|
||||||
}
|
}
|
||||||
|
|
||||||
err = c.Build(name, suse.envs(), suse.runs())
|
if !c.Exist() {
|
||||||
if err != nil {
|
err = c.Build(name, suse.envs(), suse.runs())
|
||||||
return
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd := "zypper search -s --match-exact kernel-default | grep x86_64 " +
|
cmd := "zypper search -s --match-exact kernel-default | grep x86_64 " +
|
||||||
|
@ -38,9 +38,12 @@ func (ol OracleLinux) Packages() (pkgs []string, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = c.Build("oraclelinux:"+ol.release, ol.envs(), ol.runs())
|
if !c.Exist() {
|
||||||
if err != nil {
|
err = c.Build("oraclelinux:"+ol.release,
|
||||||
return
|
ol.envs(), ol.runs())
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ol.release == "8" {
|
if ol.release == "8" {
|
||||||
|
@ -43,9 +43,11 @@ func (u Ubuntu) Packages() (pkgs []string, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
|
if !c.Exist() {
|
||||||
if err != nil {
|
err = c.Build("ubuntu:"+u.release, u.envs(), u.runs())
|
||||||
return
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd := "apt-cache search " +
|
cmd := "apt-cache search " +
|
||||||
|
Loading…
Reference in New Issue
Block a user