From 793ac0071dea61382d8ae5e6517b97720add4774 Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Wed, 22 May 2024 18:00:18 +0000 Subject: [PATCH] feat: ubuntu 24.04 --- .github/workflows/e2e.yml | 1 + .github/workflows/images-ubuntu.yml | 89 ++++++++++++++++++++++++ .github/workflows/ubuntu.yml | 1 + distro/ubuntu/ubuntu.go | 1 + tools/qemu-ubuntu-img/generate-images.py | 70 +++++++++++++++++++ tools/qemu-ubuntu-img/setup.sh | 2 +- 6 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/images-ubuntu.yml create mode 100755 tools/qemu-ubuntu-img/generate-images.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4206e8b..8fa40fc 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,6 +31,7 @@ jobs: { distro: Ubuntu, release: 18.04 }, { distro: Ubuntu, release: 20.04 }, { distro: Ubuntu, release: 22.04 }, + { distro: Ubuntu, release: 24.04 }, { distro: CentOS, release: 6 }, { distro: CentOS, release: 7 }, { distro: CentOS, release: 8 }, diff --git a/.github/workflows/images-ubuntu.yml b/.github/workflows/images-ubuntu.yml new file mode 100644 index 0000000..2071653 --- /dev/null +++ b/.github/workflows/images-ubuntu.yml @@ -0,0 +1,89 @@ +name: Ubuntu images + +on: + workflow_dispatch: + push: + paths: + - 'tools/qemu-ubuntu-img/**' + - '.github/workflows/images-ubuntu.yml' + +concurrency: + group: ${{ github.workflow_ref }} + cancel-in-progress: true + +jobs: + images: + name: Qemu Images + runs-on: ubuntu-latest + steps: + - uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + + - uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: create droplet + run: doctl compute droplet create --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94,37:46:77:a8:4a:96:3b:20:16:46:35:04:95:ca:0c:5c' --tag-name=github-actions ga-out-of-tree-images-ubuntu-$GITHUB_SHA --size s-1vcpu-1gb --image ubuntu-22-04-x64 --wait + + # TODO Move to common script + - name: generate images + shell: bash + run: | + set -x + + sleep 1m + + IP=$(doctl compute droplet list --tag-name=github-actions --format "Name,Public IPv4" | grep -v ID | grep ga-out-of-tree-images-ubuntu-$GITHUB_SHA | awk '{print $2}') + + while ! ssh -o StrictHostKeyChecking=accept-new root@$IP echo + do + sleep 1s + done + + ssh root@$IP "cloud-init status --wait | grep done" + + ssh root@$IP apt-get update + ssh root@$IP apt-get install -y git podman s3cmd + ssh root@$IP git clone https://github.com/out-of-tree/out-of-tree + ssh root@$IP "cd out-of-tree && git checkout $GITHUB_SHA" + + ssh root@$IP "echo -e '[Unit]\nDescription=Ubuntu image generator and uploader\n[Service]\nRemainAfterExit=yes\nStandardError=append:/var/log/images-ubuntu.log\nStandardOutput=append:/var/log/images-ubuntu.log\nType=oneshot' >> /etc/systemd/system/images-ubuntu.service" + + ssh root@$IP "echo Environment=HOST=fra1.digitaloceanspaces.com >> /etc/systemd/system/images-ubuntu.service" + ssh root@$IP "echo Environment=HOST_BUCKET=out-of-tree.fra1.digitaloceanspaces.com >> /etc/systemd/system/images-ubuntu.service" + ssh root@$IP "echo Environment=ACCESS_KEY=${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }} >> /etc/systemd/system/images-ubuntu.service" + ssh root@$IP "echo Environment=SECRET_KEY=${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }} >> /etc/systemd/system/images-ubuntu.service" + + ssh root@$IP "echo 'ExecStart=/root/out-of-tree/tools/qemu-ubuntu-img/generate-images.py' >> /etc/systemd/system/images-ubuntu.service" + + ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-ubuntu-img/*.tar.gz s3://out-of-tree/1.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-ubuntu.service' + + ssh root@$IP "echo TimeoutStopSec=1 >> /etc/systemd/system/images-ubuntu.service" + + ssh root@$IP systemctl daemon-reload + + ssh root@$IP systemctl start images-ubuntu --no-block + + while ! ssh root@$IP systemctl show images-ubuntu -p SubState --value | grep -E '(failed|exited)' + do + sleep 1m + done + + scp root@$IP:/var/log/images-ubuntu.log . + + cat images-ubuntu.log + + ssh root@$IP systemctl is-active images-ubuntu + + - name: Archive logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: images-ubuntu-log + path: images-ubuntu.log + + - name: delete droplet + if: always() + run: doctl compute droplet delete -f ga-out-of-tree-images-ubuntu-$GITHUB_SHA diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 1658a8e..943922b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -113,6 +113,7 @@ jobs: { distro: Ubuntu, release: 18.04 }, { distro: Ubuntu, release: 20.04 }, { distro: Ubuntu, release: 22.04 }, + { distro: Ubuntu, release: 24.04 }, { distro: CentOS, release: 6 }, { distro: CentOS, release: 7 }, { distro: CentOS, release: 8 }, diff --git a/distro/ubuntu/ubuntu.go b/distro/ubuntu/ubuntu.go index ccef937..590d7ac 100644 --- a/distro/ubuntu/ubuntu.go +++ b/distro/ubuntu/ubuntu.go @@ -17,6 +17,7 @@ func init() { "18.04", "20.04", "22.04", + "24.04", } for _, release := range releases { diff --git a/tools/qemu-ubuntu-img/generate-images.py b/tools/qemu-ubuntu-img/generate-images.py new file mode 100755 index 0000000..09eb0e1 --- /dev/null +++ b/tools/qemu-ubuntu-img/generate-images.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import os +import subprocess + +script_dir = os.path.dirname(os.path.realpath(__file__)) +os.chdir(script_dir) + +releases = [ + # ('12.04', 'precies'), + # ('14.04', 'trusty'), + # ('16.04', 'xenial'), + # ('18.04', 'bionic'), + # ('20.04', 'focal'), + # ('22.04', 'jammy'), + ('24.04', 'noble'), +] + +template = ''' +FROM ubuntu:{version} + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update +RUN apt install -y debootstrap qemu-utils +RUN apt install -y linux-image-generic + +ENV TMPDIR=/tmp/ubuntu +ENV IMAGEDIR=/tmp/image +ENV IMAGE=/shared/out_of_tree_ubuntu_{img_version}.img +ENV REPOSITORY=http://archive.ubuntu.com/ubuntu +ENV RELEASE={codename} + +RUN mkdir $IMAGEDIR + +# Must be executed with --privileged because of /dev/loop +CMD debootstrap --include=openssh-server,policykit-1 \ + $RELEASE $TMPDIR $REPOSITORY && \ + /shared/setup.sh $TMPDIR && \ + qemu-img create $IMAGE 2G && \ + mkfs.ext4 -F $IMAGE && \ + mount -o loop $IMAGE $IMAGEDIR && \ + cp -a $TMPDIR/* $IMAGEDIR/ && \ + umount $IMAGEDIR +''' + +def run_cmd(cmd): + print(f"+ {cmd}") + subprocess.run(cmd, shell=True, check=True, executable='/bin/bash') + +for version, codename in releases: + numeric_version = version.replace('.', '') + img_version=version.replace(".","__") + + dockerfile_content = template.format( + version=version, + img_version=img_version, + codename=codename, + numeric_version=numeric_version) + + os.makedirs(str(version), exist_ok=True) + with open(f"{version}/Dockerfile", "w") as dockerfile: + dockerfile.write(dockerfile_content) + + run_cmd(f"podman build -t gen-ubuntu{numeric_version}-image {version}") + run_cmd(f"rm -rf {version}") + + run_cmd(f"podman run --privileged -v {os.getcwd()}:/shared -t gen-ubuntu{numeric_version}-image") + + run_cmd(f"tar -Szcf out_of_tree_ubuntu_{img_version}.img.tar.gz out_of_tree_ubuntu_{img_version}.img") + diff --git a/tools/qemu-ubuntu-img/setup.sh b/tools/qemu-ubuntu-img/setup.sh index 6779283..03c2b4e 100755 --- a/tools/qemu-ubuntu-img/setup.sh +++ b/tools/qemu-ubuntu-img/setup.sh @@ -13,5 +13,5 @@ sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config echo '#!/bin/sh' > $TMPDIR/etc/rc.local -echo 'dhclient' >> $TMPDIR/etc/rc.local +echo 'dhclient || dhcpcd' >> $TMPDIR/etc/rc.local chmod +x $TMPDIR/etc/rc.local