From f15653b4cdc0e07c6ece1636bcdecb01b8791a5b Mon Sep 17 00:00:00 2001 From: Mo8it Date: Thu, 18 Aug 2022 03:20:52 +0200 Subject: [PATCH] Finalize container setup with data_dir --- .gitignore | 22 ++- advlabdb/__init__.py | 9 +- advlabdb/config.py | 50 ++++-- advlabdb/database_import.py | 4 +- cli/setup/generate_secrets/main.py | 4 +- cli/setup/init_db/main.py | 4 +- cli/test/generate_test_db/main.py | 4 +- podman/deploy.py | 242 ++++++++++++++++------------- settings_example.ini | 1 - 9 files changed, 197 insertions(+), 143 deletions(-) diff --git a/.gitignore b/.gitignore index 7d4bd39..f003d00 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,16 @@ -# Do not commit/publish the secrets file with the secret key and password salt! +# Do not commit/share/publish the secrets file with the secret key and password salt! secrets.ini - # Own settings settings.ini - -# Python -__pycache__ -*.pyc - # Database -db/ - -# Poetry -.venv/ +*.db # Flask-Migrate -migrations/ +/migrations/ + +# Development +/dev_data/ + +# Python +__pycache__/ +*.pyc diff --git a/advlabdb/__init__.py b/advlabdb/__init__.py index a3d37cb..28afe34 100644 --- a/advlabdb/__init__.py +++ b/advlabdb/__init__.py @@ -1,17 +1,18 @@ from flask_migrate import Migrate from flask_security.datastore import SQLAlchemyUserDatastore -from .config import get_settings +from .config import get_settings, get_data_dir from .models import db, User, Role migrate = Migrate() -settings = get_settings() +data_dir = get_data_dir() +settings = get_settings(data_dir) user_datastore = SQLAlchemyUserDatastore(db, User, Role) -def create_app(create_for_server=True): +def create_app(create_for_server: bool = True): from flask import Flask app = Flask(__name__) @@ -19,7 +20,7 @@ def create_app(create_for_server=True): # Config from .config import set_config - set_config(app) + set_config(app, data_dir) # Setup Flask-SQLAlchemy diff --git a/advlabdb/config.py b/advlabdb/config.py index 8bc70e8..319fd12 100644 --- a/advlabdb/config.py +++ b/advlabdb/config.py @@ -1,39 +1,59 @@ import sys from configparser import ConfigParser +from os import environ from pathlib import Path -def load_config(*files): +def get_data_dir() -> Path: + data_dir_env_variable = "ADVLABDB_DATA_DIR" + data_dir = Path(environ.get(data_dir_env_variable, "dev_data")) + + if not data_dir.is_dir(): + sys.exit( + f""" +You did not set the environment variable {data_dir_env_variable} which is the path to the directory which holds the data of AdvLabDB including the configuration. +Read the documentation for more information! +""" + ) + + return data_dir + + +def load_config(file_name: str, data_dir: Path): config = ConfigParser() - for file in files: - file = Path(file) + file = data_dir / file_name - if not file.is_file(): - print(f"{file} is missing!") - sys.exit(1) + if not file.is_file(): + sys.exit(f"{file} is missing!") - config.read(file) + config.read(file) return config -def get_settings(): - config = load_config("settings.ini") - settings = config["Settings"] +def get_secrets(data_dir: Path): + config = load_config("secrets.ini", data_dir) + secrets = settings_config["Secrets"] + + return secrets + + +def get_settings(data_dir: Path): + config = load_config("settings.ini", data_dir) + settings = settings_config["Settings"] return settings -def set_config(app): - config = load_config("secrets.ini", "settings.ini") - secrets = config["Secrets"] - settings = config["Settings"] +def set_config(app, data_dir: Path): + secrets = get_secrets(data_dir) + settings = get_settings(data_dir) app.config["SECRET_KEY"] = secrets["SECRET_KEY"] # SQLALCHEMY - db_file = Path(settings["SQLITE_DB_PATH"]) + db_file = data_dir / "db/advlab.db" db_file.parent.mkdir(parents=True, exist_ok=True) app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{db_file}" diff --git a/advlabdb/database_import.py b/advlabdb/database_import.py index 29974de..a00605c 100644 --- a/advlabdb/database_import.py +++ b/advlabdb/database_import.py @@ -5,7 +5,7 @@ from shutil import copy2 from flask import flash from sqlalchemy import select -from . import settings +from . import data_dir from .exceptions import DataBaseImportException from .models import ( Appointment, @@ -48,7 +48,7 @@ def not_nullable(entry): def importFromFile(filePath): - db_path = Path(settings["SQLITE_DB_PATH"]) + db_path = data_dir / "db/advlab.db" db_bk_dir = db_path.parent / "backups" db_bk_dir.mkdir(exist_ok=True) diff --git a/cli/setup/generate_secrets/main.py b/cli/setup/generate_secrets/main.py index a0dca69..78eb022 100644 --- a/cli/setup/generate_secrets/main.py +++ b/cli/setup/generate_secrets/main.py @@ -3,9 +3,11 @@ from pathlib import Path import click +from advlabdb import data_dir + def _generate_secrets(): - file = Path("secrets.ini") + file = data_dir / "secrets.ini" if file.is_file(): click.echo(f"Skipping secrets generation because the secrets file does already exist at {file}.") diff --git a/cli/setup/init_db/main.py b/cli/setup/init_db/main.py index 94661cd..faf74e9 100644 --- a/cli/setup/init_db/main.py +++ b/cli/setup/init_db/main.py @@ -4,7 +4,7 @@ import click from email_validator import validate_email from flask_security.utils import hash_password -from advlabdb import create_app, settings, user_datastore +from advlabdb import create_app, data_dir, user_datastore from advlabdb.model_independent_funs import randomPassword from advlabdb.models import MAX_YEAR, MIN_YEAR, Admin, Semester, db @@ -19,7 +19,7 @@ class EmailParamType(click.ParamType): def _init_db(manage): - db_file = Path(settings["SQLITE_DB_PATH"]) + db_file = data_dir / "db/advlab.db" if db_file.is_file(): click.echo(f"Skipping database initialization because the database does already exist at {db_file}.") return diff --git a/cli/test/generate_test_db/main.py b/cli/test/generate_test_db/main.py index 7fb3d17..6e1a772 100644 --- a/cli/test/generate_test_db/main.py +++ b/cli/test/generate_test_db/main.py @@ -6,7 +6,7 @@ from random import randint, random import click from flask_security.utils import hash_password -from advlabdb import create_app, settings, user_datastore +from advlabdb import create_app, data_dir, user_datastore from advlabdb.exceptions import DatabaseException from advlabdb.models import ( Admin, @@ -34,7 +34,7 @@ def db_add(obj): def _generate_test_db(): - db_file = Path(settings["SQLITE_DB_PATH"]) + db_file = data_dir / "db/advlab.db" if db_file.is_file(): click.echo( click.style( diff --git a/podman/deploy.py b/podman/deploy.py index 4d48752..7c178ff 100755 --- a/podman/deploy.py +++ b/podman/deploy.py @@ -13,12 +13,9 @@ VOLUMES_DIR = Path.home() / "volumes" # AdvLabDB ADVLABDB_REPO_LINK = "https://gitlab.rlp.net/mobitar/advlabdb.git" ADVLABDB_VOLUMES_DIR = VOLUMES_DIR / "advlabdb" -# TODO: Do not mount the repo. -# TODO: Mount volume for data. -# TODO: Use environment variable for config dir. -# TODO: Mount volume for settings and secrets. ADVLABDB_REPO_DIR = ADVLABDB_VOLUMES_DIR / "repo" ADVLABDB_LOGS_DIR = ADVLABDB_VOLUMES_DIR / "logs" +ADVLABDB_DATA_DIR = ADVLABDB_VOLUMES_DIR / "data" # Traefik TRAEFIK_VOLUMES_DIR = VOLUMES_DIR / "traefik" TRAEFIK_ETC_DIR = TRAEFIK_VOLUMES_DIR / "etc" @@ -90,7 +87,7 @@ def create_container(container_name: str, podman_args: str): ) print(f"Generating a systemd service then enabling and starting it for the container {container_name}.") - print(f"The service is a user service and named container-{container_name}.") + print(f"The service is a user service named container-{container_name}.") print("You can check its status with the following command:") print(f"\tsystemctl --user status container-{container_name}") run( @@ -104,112 +101,53 @@ def create_container(container_name: str, podman_args: str): ) -# Checking requirements - -settings_file = ADVLABDB_REPO_DIR / "settings.ini" - -if ADVLABDB_REPO_DIR.is_dir(): - print("Pulling AdvLabDB repository.") - run( - "git pull origin main", - cwd=ADVLABDB_REPO_DIR, - check=True, - ) - - if not settings_file.is_file(): - sys.exit(f"{settings_file} missing!") -else: - if args.verbose: - print(f"Making sure that the volumes directory {ADVLABDB_VOLUMES_DIR} exists.") - ADVLABDB_VOLUMES_DIR.mkdir(parents=True, exist_ok=True) - - print("Cloning AdvLabDB repository") - print(f"From:\t{ADVLABDB_REPO_LINK}") - print(f"Into:\t{ADVLABDB_REPO_DIR}") - run( - f"git clone {ADVLABDB_REPO_LINK} {ADVLABDB_REPO_DIR}", - check=True, - ) - sys.exit(f"{settings_file} missing!") - -if not args.skip_traefik: - if not TRAEFIK_ETC_DIR.is_dir(): - sys.exit(f"{TRAEFIK_ETC_DIR} missing!") - -if not args.skip_nginx: - if not NGINX_CONF_D_DIR.is_dir(): - sys.exit(f"{NGINX_CONF_D_DIR} missing!") - -if run(f"podman network exists {args.network}").returncode != 0: - if args.skip_traefik: - sys.exit(f"Skipped Traefik's deployment although Traefik's network {args.network} does not exist!") - else: - print(f"Creating network {args.network}.") +def pull_or_clone_repo(): + if ADVLABDB_REPO_DIR.is_dir(): + print("Pulling AdvLabDB repository.") run( - f"podman network create {args.network}", + "git pull --rebase origin main", + cwd=ADVLABDB_REPO_DIR, + check=True, + ) + else: + if args.verbose: + print(f"Making sure that the volumes directory {ADVLABDB_VOLUMES_DIR} exists.") + ADVLABDB_VOLUMES_DIR.mkdir(parents=True, exist_ok=True) + + print("Cloning AdvLabDB repository") + print(f"From:\t{ADVLABDB_REPO_LINK}") + print(f"Into:\t{ADVLABDB_REPO_DIR}") + run( + f"git clone {ADVLABDB_REPO_LINK} {ADVLABDB_REPO_DIR}", check=True, ) -# Create/update the AdvLabDB image and container -# Make sure that the builder container does not exist. -run( - "buildah rm builder", - stderr=subprocess.DEVNULL, -) +def check_requirements(): + settings_file = ADVLABDB_DATA_DIR / "settings.ini" + if not settings_file.is_file(): + sys.exit(f"{settings_file} missing!") -print("Creating AdvLabDB image.") -commands = [ - "buildah from --pull --name builder docker.io/library/python:3.10-slim", - # Copy repo into container - f"buildah copy builder {ADVLABDB_REPO_DIR} /volumes/repo", - "buildah config --workingdir /volumes/repo builder", - # Install Python requirements in the container - "buildah run builder -- pip3 install -r requirements.txt", - "buildah run builder -- python3 manage.py setup generate-secrets", - "buildah run builder -- python3 manage.py setup init-db", - "buildah config --cmd 'gunicorn --bind 0.0.0.0:80 --workers 5 --log-file /volumes/logs/gunicorn.log run:app' builder", -] + if not args.skip_traefik: + if not TRAEFIK_ETC_DIR.is_dir(): + sys.exit(f"{TRAEFIK_ETC_DIR} missing!") -for command in commands: - run(command, check=True) + if not args.skip_nginx: + if not NGINX_CONF_D_DIR.is_dir(): + sys.exit(f"{NGINX_CONF_D_DIR} missing!") -if run("systemctl --user is-enabled container-advlabdb").returncode == 0: - print("Disabling and deleting existing container advlabdb.") - run( - "systemctl --user disable --now container-advlabdb", - check=True, - ) + if run(f"podman network exists {args.network}").returncode != 0: + if args.skip_traefik: + sys.exit(f"Skipped Traefik's deployment although Traefik's network {args.network} does not exist!") + else: + print(f"Creating network {args.network}.") + run( + f"podman network create {args.network}", + check=True, + ) -if run("podman image exists advlabdb").returncode == 0: - print("Deleting existing image advlabdb") - run( - "podman rmi advlabdb", - check=True, - ) -# Save new image -run( - "buildah commit --rm builder advlabdb", - check=True, -) - -if args.verbose: - print(f"Making sure that the logs directory {ADVLABDB_LOGS_DIR} exists.") -ADVLABDB_LOGS_DIR.mkdir(parents=True, exist_ok=True) - -print("Creating container advlabdb.") -create_container( - "advlabdb", - # TODO: Add database as volume - f"""--network {args.network} \ - -v {ADVLABDB_LOGS_DIR}:/volumes/logs:Z \ - localhost/advlabdb:latest""", -) - -# Create Traefik container if needed - -if not args.skip_traefik and run("systemctl --user is-enabled container-traefik").returncode != 0: +def create_traefik_container(): if args.verbose: print(f"Making sure that the logs directory {TRAEFIK_LOGS_DIR} exists.") TRAEFIK_LOGS_DIR.mkdir(parents=True, exist_ok=True) @@ -231,9 +169,8 @@ if not args.skip_traefik and run("systemctl --user is-enabled container-traefik" docker.io/library/traefik:latest""", ) -# Create Nginx container if needed -if not args.skip_nginx and run("systemctl --user is-enabled container-nginx").returncode != 0: +def create_nginx_container(): print("Creating container nginx.") create_container( "nginx", @@ -243,4 +180,101 @@ if not args.skip_nginx and run("systemctl --user is-enabled container-nginx").re docker.io/library/nginx:alpine""", ) -print("\nDone!\n") + +def main(): + pull_or_clone_repo() + + # Checking requirements + + check_requirements() + + # Create/update the AdvLabDB image and container + + # Make sure that the builder container does not exist. + run( + "buildah rm builder", + stderr=subprocess.DEVNULL, + ) + + requirements_file = ADVLABDB_REPO_DIR / "requirements.txt" + + print("Creating AdvLabDB image.") + commands = ( + # Start building from a base image + "buildah from --pull --name builder docker.io/library/python:3.10-slim", + # Install Python requirements in the container + f"buildah copy builder {requirements_file} /root/requirements.txt" + "buildah run builder -- pip3 install -r /root/requirements.txt", + "buildah run builder -- rm /root/requirements.txt", + # Set the working directory of the container + "buildah config --workingdir /volumes/repo builder", + # Set the command that will run after starting the container + "buildah config --cmd 'gunicorn --bind 0.0.0.0:80 --workers 5 --log-file /volumes/logs/gunicorn.log run:create_app()' builder", + ) + + for command in commands: + run(command, check=True) + + if run("systemctl --user is-enabled container-advlabdb").returncode == 0: + print("Disabling and deleting existing container advlabdb.") + run( + "systemctl --user disable --now container-advlabdb", + check=True, + ) + + if run("podman image exists localhost/advlabdb:latest").returncode == 0: + print("Deleting existing image advlabdb.") + run( + "podman rmi localhost/advlabdb:latest", + check=True, + ) + + # Save new image + run( + "buildah commit --rm builder advlabdb", + check=True, + ) + + if args.verbose: + print(f"Making sure that the logs directory {ADVLABDB_LOGS_DIR} exists.") + ADVLABDB_LOGS_DIR.mkdir(parents=True, exist_ok=True) + + container_args = f"""--network {args.network} \ + -e ADVLABDB_DATA_DIR=/volumes/data \ + -v {ADVLABDB_REPO_DIR}:/volumes/repo:Z \ + -v {ADVLABDB_DATA_DIR}:/volumes/data:Z \ + -v {ADVLABDB_LOGS_DIR}:/volumes/logs:Z""" + + # Running setup commands (if needed) + run_manage = f"podman run -it --rm {container_args} localhost/advlabdb:latest python3 manage.py" + commands = ( + # Generate secret keys if secrets.ini does not exist yet + f"{run_manage} setup generate-secrets", + # Initialize a database if none exists + f"{run_manage} setup init-db", + ) + + for command in commands: + run(command, check=True) + + print("Creating container advlabdb.") + create_container( + "advlabdb", + f"{container_args} localhost/advlabdb:latest", + ) + + # Create Traefik container if needed + + if not args.skip_traefik and run("systemctl --user is-enabled container-traefik").returncode != 0: + create_traefik_container() + + # Create Nginx container if needed + + if not args.skip_nginx and run("systemctl --user is-enabled container-nginx").returncode != 0: + create_nginx_container() + + print("\nDone!\n") + + +if __name__ == "__main__": + main() diff --git a/settings_example.ini b/settings_example.ini index b60a2f6..38ed95d 100644 --- a/settings_example.ini +++ b/settings_example.ini @@ -1,4 +1,3 @@ [Settings] -SQLITE_DB_PATH = /volumes/data/advlabdb.db CHECK_EMAIL_DELIVERABILITY = True SECURITY_PASSWORD_LENGTH_MIN = 15