From 50fd9ae77de2a9a823d34bfd16c815d5e2ec2998 Mon Sep 17 00:00:00 2001 From: Mo8it Date: Tue, 2 Aug 2022 13:39:44 +0200 Subject: [PATCH] Add deploy script for containers --- .pre-commit-config.yaml | 3 +- advlabdb/scripts/test/test_container.py | 103 --------------------- podman/deploy.py | 115 ++++++++++++++++++------ 3 files changed, 91 insertions(+), 130 deletions(-) delete mode 100644 advlabdb/scripts/test/test_container.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90fb50c..a19d875 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: - id: bandit - repo: https://github.com/pycqa/flake8 - rev: 5.0.0 + rev: 5.0.3 hooks: - id: flake8 @@ -44,5 +44,4 @@ repos: rev: 1.2.0b3 hooks: - id: poetry-check - - id: poetry-lock - id: poetry-export diff --git a/advlabdb/scripts/test/test_container.py b/advlabdb/scripts/test/test_container.py deleted file mode 100644 index aa20ee0..0000000 --- a/advlabdb/scripts/test/test_container.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -This script builds a Debian container to test the server setup. - -Requirements: - * python3 - * podman - * buildah - -On SELinux systems, you have to run the following to be able to run containers with systemd inside (taken from `man podman run`): - `sudo setsebool -P container_manage_cgroup true` - -Run this script simply as a Python script: - `python3 test_container.py` - -You will need to edit the file `.env`. -Enter anything for the `SECRET_KEY` and `SECURITY_PASSWORD_SALT` (only for testing!). -Then set `SERVER_NAME=127.0.0.1:8080`. - -The script will throw some errors related to systemd. -This is the case on `systemctl reboot` for example. -These errors should not be relevant. -They occur because of building an image and not running commands in a live server / container. - -After running the script, visit `http://127.0.0.1:8080` in your browser. - -If you want to clean up after testing with the container, run the following: - `podman kill advlabdb` - `podman rm advlabdb` - `podman rmi advlabdb` - `podman rmi systemd_debian` -""" - -import subprocess # nosec 404 - - -def run(command, **kwargs): - return subprocess.run(command, shell=True, **kwargs) # nosec B602 - - -def update_system(container_name): - return run(f"buildah run {container_name} -- apt update && apt dist-upgrade -y && apt autoremove -y") - - -def run_a(command): - return run(f"buildah run advlabdb -- {command}") - - -def main(): - print("<> Remove old Buildah containers") - run("buildah rm -a") - - if run("podman image exists systemd_debian").returncode != 0: - print("<> Build debian image with systemd") - run("buildah from --name tmp docker.io/library/debian:latest") - update_system("tmp") - run("buildah run tmp -- apt install -y systemd systemd-sysv fish neovim fd-find ripgrep") - run("buildah config --cmd /sbin/init tmp") - print("<> Commit debian image with systemd") - run("buildah commit --rm tmp systemd_debian") - - if run("podman container exists advlabdb").returncode == 0: - print("<> Remove old advlabdb container") - run("podman kill advlabdb") - run("podman rm advlabdb") - - print("<> Build advlabdb image") - run("buildah from --name advlabdb localhost/systemd_debian:latest") - print("<> Root setup") - update_system("advlabdb") - run_a("apt install sudo python3 git -y") - run_a("sudo useradd admin") - run_a("sudo usermod -aG sudo admin") - run_a("mkhomedir_helper admin") - print("Enter new admin user password:") - run_a("sudo passwd admin") - - print("<> Admin setup") - run("buildah config --workingdir /home/admin -u admin advlabdb") - run_a("git clone https://gitlab.rlp.net/mobitar/advlabdb.git") - run("buildah config --workingdir /home/admin/advlabdb advlabdb") - run_a("cp -v advlabdb/scripts/setup/advlabdb.conf.template advlabdb/scripts/setup/advlabdb.conf") - run_a("cp -v .env.template .env") - run_a("nvim .env") - run_a("python3 advlabdb/scripts/setup/server_setup.py") - run_a("/home/admin/.local/bin/poetry run python3 -m advlabdb.scripts.setup.init_database") - - run("buildah config -u root advlabdb") - - if run("podman image exists advlabdb").returncode == 0: - print("<> Remove old image") - run("podman rmi advlabdb") - - print("<> Commit image advlabdb") - run("buildah commit --rm advlabdb advlabdb") - - print("<> Start container") - run("podman run -dit --name advlabdb -p 8080:80 advlabdb:latest") - - print("<> Done! Now visit http://127.0.0.1:8080") - - -if __name__ == "__main__": - main() diff --git a/podman/deploy.py b/podman/deploy.py index 73103ef..2ba8d3b 100755 --- a/podman/deploy.py +++ b/podman/deploy.py @@ -5,27 +5,78 @@ import sys from argparse import ArgumentParser from pathlib import Path -ADVLABDB_REPO_LINK = "https://gitlab.rlp.net/mobitar/advlabdb.git" +# Variables +# You might want to change some. + +# Volumes VOLUMES_DIR = Path.home() / "volumes" +# AdvLabDB +ADVLABDB_REPO_LINK = "https://gitlab.rlp.net/mobitar/advlabdb.git" ADVLABDB_VOLUMES_DIR = VOLUMES_DIR / "advlabdb" ADVLABDB_REPO_DIR = ADVLABDB_VOLUMES_DIR / "repo" ADVLABDB_LOGS_DIR = ADVLABDB_VOLUMES_DIR / "logs" +# Traefik TRAEFIK_VOLUMES_DIR = VOLUMES_DIR / "traefik" TRAEFIK_ETC_DIR = TRAEFIK_VOLUMES_DIR / "etc" -NGINX_CONF_D_DIR = VOLUMES_DIR / "nginx/conf.d" TRAEFIK_LOGS_DIR = TRAEFIK_VOLUMES_DIR / "logs" TRAEFIK_CERTS_DIR = TRAEFIK_VOLUMES_DIR / "certs" +# Nginx +NGINX_CONF_D_DIR = VOLUMES_DIR / "nginx/conf.d" +# Systemd SYSTEMD_USER_DIR = Path.home() / ".config/systemd/user" +# Parse script arguments + +parser = ArgumentParser( + description="Build the AdvLabDB image and deploy related containers.", +) +parser.add_argument( + "--skip-traefik", + action="store_true", + help="Do not deploy the Traefik container.", +) +parser.add_argument( + "--skip-nginx", + action="store_true", + help="Do not deploy the Nginx container.", +) +parser.add_argument( + "-n", + "--network", + default="traefik", + help="Traefik's network. Default: traefik.", +) +parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Show more information while running.", +) + +args = parser.parse_args() + +# Functions + + def run(command: str, **kwargs): + if args.verbose: + print("Running command:") + print(f"\t{command}") return subprocess.run(command, shell=True, **kwargs) # nosec B602 +# Needed for create_container +if args.verbose: + print(f"Making sure that the systemd directory for user services {SYSTEMD_USER_DIR} exists.") SYSTEMD_USER_DIR.mkdir(parents=True, exist_ok=True) def create_container(container_name: str, podman_args: str): + """ + Create a container with a systemd service. + """ + print(f"Creating container: {container_name}.") run( f"""podman create \ --name {container_name} \ @@ -33,6 +84,10 @@ def create_container(container_name: str, podman_args: str): check=True, ) + print(f"Generating a systemd service then enabling and starting it for the container {container_name}.") + print(f"The service is a user service and named container-{container_name}.") + print("You can check its status with the following command:") + print(f"\tsystemctl --user status container-{container_name}") run( f"podman generate systemd --new --files --name {container_name}", cwd=SYSTEMD_USER_DIR, @@ -44,31 +99,12 @@ def create_container(container_name: str, podman_args: str): ) -parser = ArgumentParser( - description="Build the AdvLabDB image and deploy related containers", -) -parser.add_argument( - "--skip-traefik", - action="store_true", - help="Do not deploy the Traefik container", -) -parser.add_argument( - "--skip-nginx", - action="store_true", - help="Do not deploy the Nginx container", -) -parser.add_argument( - "-n", - "--network", - default="traefik", - help="Traefik's network. Default: traefik", -) - -args = parser.parse_args() +# Checking requirements env_file = ADVLABDB_REPO_DIR / ".env" if ADVLABDB_REPO_DIR.is_dir(): + print("Pulling AdvLabDB repository.") run( "git pull", cwd=ADVLABDB_REPO_DIR, @@ -78,8 +114,13 @@ if ADVLABDB_REPO_DIR.is_dir(): if not env_file.is_file(): sys.exit(f"{env_file} missing!") else: + if args.verbose: + print(f"Making sure that the volumes directory {ADVLABDB_VOLUMES_DIR} exists.") ADVLABDB_VOLUMES_DIR.mkdir(parents=True, exist_ok=True) - print("Cloning repository") + + print("Cloning AdvLabDB repository") + print(f"From:\t{ADVLABDB_REPO_LINK}") + print(f"Into:\t{ADVLABDB_REPO_DIR}") run( f"git clone {ADVLABDB_REPO_LINK} {ADVLABDB_REPO_DIR}", check=True, @@ -98,21 +139,28 @@ if run(f"podman network exists {args.network}").returncode != 0: if args.skip_traefik: sys.exit(f"Skipped Traefik's deployment although Traefik's network {args.network} does not exist!") else: + print(f"Creating network {args.network}.") run( f"podman network create {args.network}", check=True, ) +# Create/update the AdvLabDB image and container + +# Make sure that the builder container does not exist. run( "buildah rm builder", stderr=subprocess.DEVNULL, ) +print("Creating AdvLabDB image.") commands = [ + # TODO: Use fixed version tag 3.10-slim "buildah from --pull --name builder docker.io/library/python:slim", - "buildah config --workingdir / builder", + # Copy repo into container f"buildah copy builder {ADVLABDB_REPO_DIR} /volumes/repo", "buildah config --workingdir /volumes/repo builder", + # Install Python requirements in the container "buildah run builder -- pip3 install -r requirements.txt", # TODO: Only when database does not exist "buildah run builder -- python3 -m advlabdb.scripts.setup.init_database", @@ -123,24 +171,30 @@ for command in commands: run(command, check=True) if run("systemctl --user is-enabled container-advlabdb").returncode == 0: + print("Disabling and deleting existing container advlabdb.") run( "systemctl --user disable --now container-advlabdb", check=True, ) if run("podman image exists advlabdb").returncode == 0: + print("Deleting existing image advlabdb") run( "podman rmi advlabdb", check=True, ) +# Save new image run( "buildah commit --rm builder advlabdb", check=True, ) +if args.verbose: + print(f"Making sure that the logs directory {ADVLABDB_LOGS_DIR} exists.") ADVLABDB_LOGS_DIR.mkdir(parents=True, exist_ok=True) +print("Creating container advlabdb.") create_container( "advlabdb", # TODO: Add database as volume @@ -149,10 +203,18 @@ create_container( localhost/advlabdb:latest""", ) +# Create Traefik container if needed + if not args.skip_traefik and run("systemctl --user is-enabled container-traefik").returncode != 0: + if args.verbose: + print(f"Making sure that the logs directory {TRAEFIK_LOGS_DIR} exists.") TRAEFIK_LOGS_DIR.mkdir(parents=True, exist_ok=True) + + if args.verbose: + print(f"Making sure that the certificates directory {TRAEFIK_CERTS_DIR} exists.") TRAEFIK_CERTS_DIR.mkdir(parents=True, exist_ok=True) + print("Creating container traefik.") create_container( "traefik", f"""--label "io.containers.autoupdate=registry" \ @@ -165,7 +227,10 @@ if not args.skip_traefik and run("systemctl --user is-enabled container-traefik" docker.io/library/traefik:latest""", ) +# Create Nginx container if needed + if not args.skip_nginx and run("systemctl --user is-enabled container-nginx").returncode != 0: + print("Creating container nginx.") create_container( "nginx", f"""--label "io.containers.autoupdate=registry" \