From 3316ec02732595276cce1396c87915b0ecf856ed Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 21 May 2020 17:03:37 +0200 Subject: [PATCH] Add focker volume remove, add lock release for pre-/post-build hooks, add better gateway example that doesn't require rebuilding of images every time. --- example/gateway/certbot/Fockerfile | 2 -- example/gateway/focker-compose.yml | 33 +++++++++++++++++-- .../gateway/gateway-cookiecutter/Fockerfile | 18 ++++++++++ .../gateway/gateway-cookiecutter/files/run.sh | 17 ++++++++++ example/gateway/nginx-http/Fockerfile | 24 +------------- example/gateway/nginx-http/files/nginx.conf | 2 +- example/gateway/nginx-https/Fockerfile | 28 ++++------------ .../{ => nginx-https}/files/crontab_root | 0 focker/compose.py | 23 +++++++++---- focker/focker.py | 8 ++++- focker/volume.py | 11 +++++++ 11 files changed, 109 insertions(+), 57 deletions(-) create mode 100644 example/gateway/gateway-cookiecutter/Fockerfile create mode 100644 example/gateway/gateway-cookiecutter/files/run.sh rename example/gateway/{ => nginx-https}/files/crontab_root (100%) diff --git a/example/gateway/certbot/Fockerfile b/example/gateway/certbot/Fockerfile index 2de9aac..881bb4e 100644 --- a/example/gateway/certbot/Fockerfile +++ b/example/gateway/certbot/Fockerfile @@ -13,8 +13,6 @@ steps: /certbot/scripts/certbot.py ] - [ files/crontab_nobody, /root/crontab_nobody ] - - [ ../files/cookiecutter.json, - /certbot/data/metadata.json ] - run: - crontab -u nobody /root/crontab_nobody - rm -v /root/crontab_nobody diff --git a/example/gateway/focker-compose.yml b/example/gateway/focker-compose.yml index d01c3fb..600b678 100644 --- a/example/gateway/focker-compose.yml +++ b/example/gateway/focker-compose.yml @@ -1,5 +1,17 @@ exec.prebuild: + - focker volume remove --force gateway-cookiecutter + + +exec.postbuild: - python3 getmetadata.py + - | + focker jail oneshot -m `pwd`/files:/cookiecutter/input/meta \ + `pwd`/nginx-http/files:/cookiecutter/input/nginx-http \ + `pwd`/nginx-https/files:/cookiecutter/input/nginx-https \ + gateway-cookiecutter:/cookiecutter/output \ + -- \ + gateway-cookiecutter \ + /cookiecutter/scripts/run.sh volumes: @@ -9,13 +21,17 @@ volumes: zfs: quota: 1G certbot-webroot: - chown: 65534:65534 + chown: 65534:80 chmod: 0750 zfs: quota: 1G + gateway-cookiecutter: + zfs: + quota: 1G images: + gateway-cookiecutter: ./gateway-cookiecutter nginx-http: ./nginx-http nginx-https: ./nginx-https certbot: ./certbot @@ -26,7 +42,12 @@ jails: image: nginx-http mounts: certbot-webroot: /certbot/webroot + gateway-cookiecutter: /cookiecutter ip4.addr: 127.0.12.1 + exec.start: | + cp -v /cookiecutter/nginx-http/nginx.conf \ + /usr/local/etc/nginx/nginx.conf && \ + /bin/sh /etc/rc certbot: image: certbot @@ -34,7 +55,10 @@ jails: mounts: certbot-data: /usr/local/etc/letsencrypt certbot-webroot: /certbot/webroot + gateway-cookiecutter: /cookiecutter exec.start: | + cp -v /cookiecutter/meta/cookiecutter.json \ + /certbot/data/metadata.json && \ if [ ! -f /certbot/data/.ready ]; then rm -vf /usr/local/etc/letsencrypt/.ready && \ /usr/local/bin/python3 /certbot/scripts/certbot.py && \ @@ -51,7 +75,10 @@ jails: - nginx-http mounts: certbot-data: /usr/local/etc/letsencrypt + gateway-cookiecutter: /cookiecutter exec.start: | - ( until [ -f /usr/local/etc/letsencrypt/.ready ]; do sleep 1; done && \ - /bin/sh /etc/rc) & + cp -v /cookiecutter/nginx-https/nginx.conf \ + /usr/local/etc/nginx/nginx.conf && \ + (( until [ -f /usr/local/etc/letsencrypt/.ready ]; do sleep 1; done && \ + /bin/sh /etc/rc) &) ip4.addr: 127.0.14.1 diff --git a/example/gateway/gateway-cookiecutter/Fockerfile b/example/gateway/gateway-cookiecutter/Fockerfile new file mode 100644 index 0000000..3e8b6a8 --- /dev/null +++ b/example/gateway/gateway-cookiecutter/Fockerfile @@ -0,0 +1,18 @@ +base: freebsd-latest + +steps: + - run: + - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg install py37-cookiecutter + - mkdir -p /cookiecutter/input/meta + - mkdir -p /cookiecutter/input/nginx-http + - mkdir -p /cookiecutter/input/nginx-https + - mkdir -p /cookiecutter/templates/nginx-http/\{\{cookiecutter.directory_name\}\} + - mkdir -p /cookiecutter/templates/nginx-https/\{\{cookiecutter.directory_name\}\} + - mkdir -p /cookiecutter/output/nginx-http + - mkdir -p /cookiecutter/output/nginx-https + - mkdir -p /cookiecutter/scripts + - copy: + - [ files/run.sh, + /cookiecutter/scripts/run.sh ] + - run: + - chmod a+x /cookiecutter/scripts/run.sh diff --git a/example/gateway/gateway-cookiecutter/files/run.sh b/example/gateway/gateway-cookiecutter/files/run.sh new file mode 100644 index 0000000..5bf9f4d --- /dev/null +++ b/example/gateway/gateway-cookiecutter/files/run.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +cp -v /cookiecutter/input/nginx-http/nginx.conf \ + /cookiecutter/templates/nginx-http/\{\{cookiecutter.directory_name\}\}/nginx.conf +cp -v /cookiecutter/input/meta/cookiecutter.json \ + /cookiecutter/templates/nginx-http/cookiecutter.json + +cp -v /cookiecutter/input/nginx-https/nginx.conf \ + /cookiecutter/templates/nginx-https/\{\{cookiecutter.directory_name\}\}/nginx.conf +cp -v /cookiecutter/input/meta/cookiecutter.json \ + /cookiecutter/templates/nginx-https/cookiecutter.json + +cd /cookiecutter/output +cookiecutter --no-input /cookiecutter/templates/nginx-http directory_name=nginx-http + +cd /cookiecutter/output +cookiecutter --no-input /cookiecutter/templates/nginx-https directory_name=nginx-https diff --git a/example/gateway/nginx-http/Fockerfile b/example/gateway/nginx-http/Fockerfile index 101e167..5dab8a9 100644 --- a/example/gateway/nginx-http/Fockerfile +++ b/example/gateway/nginx-http/Fockerfile @@ -2,31 +2,9 @@ base: freebsd-latest steps: - run: - - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg install python3 nginx py37-cookiecutter - - copy: - - [ ../files/cookiecutter.json, - /root/cookiecutter.json ] - - [ files/nginx.conf, - /root/nginx.conf ] - - [ ../files/crontab_root, - /root/crontab_root ] - - run: - - mkdir -p /root/nginx_template/\{\{cookiecutter.directory_name\}\} - - mv -v /root/nginx.conf /root/nginx_template/\{\{cookiecutter.directory_name\}\}/nginx.conf - - mv -v /root/cookiecutter.json /root/nginx_template/cookiecutter.json - - run: - - cookiecutter --no-input /root/nginx_template - - mv -v ./nginx_conf/nginx.conf /usr/local/etc/nginx/nginx.conf - - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg remove python3 py37-cookiecutter - - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg autoremove - - rm -rvf /root/nginx_template - - rm -rvf ./nginx_conf + - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg install nginx - mkdir -p /certbot/webroot - - run: - sysrc nginx_enable=YES - - crontab -u root /root/crontab_root - - rm -v /root/crontab_root - - run: - sysrc sshd_enable=NO - sysrc sendmail_enable=NONE - sysrc clear_tmp_enable=YES diff --git a/example/gateway/nginx-http/files/nginx.conf b/example/gateway/nginx-http/files/nginx.conf index 26ce672..30e4767 100644 --- a/example/gateway/nginx-http/files/nginx.conf +++ b/example/gateway/nginx-http/files/nginx.conf @@ -14,7 +14,7 @@ http { server_name {{ ' '.join(cookiecutter.domains[i]) }}; location /.well-known/ { - root /srv/certbot-webroot; + root /certbot/webroot; } location / { diff --git a/example/gateway/nginx-https/Fockerfile b/example/gateway/nginx-https/Fockerfile index 3031316..b2a4fec 100644 --- a/example/gateway/nginx-https/Fockerfile +++ b/example/gateway/nginx-https/Fockerfile @@ -1,33 +1,19 @@ base: freebsd-latest steps: - - run: - - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg install python3 nginx py37-cookiecutter - copy: - - [ ../files/cookiecutter.json, - /root/cookiecutter.json ] - - [ files/nginx.conf, - /root/nginx.conf ] - - [ ../files/crontab_root, + - [ files/crontab_root, /root/crontab_root ] - run: - - mkdir -p /root/nginx_template/\{\{cookiecutter.directory_name\}\} - - mv -v /root/nginx.conf /root/nginx_template/\{\{cookiecutter.directory_name\}\}/nginx.conf - - mv -v /root/cookiecutter.json /root/nginx_template/cookiecutter.json - - run: - - cookiecutter --no-input /root/nginx_template - - mv -v ./nginx_conf/nginx.conf /usr/local/etc/nginx/nginx.conf - - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg remove python3 py37-cookiecutter - - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg autoremove - - rm -rvf /root/nginx_template - - rm -rvf ./nginx_conf + - ASSUME_ALWAYS_YES=yes IGNORE_OSVERSION=yes pkg install nginx + - mkdir -p /usr/local/etc/letsencrypt - mkdir -p /certbot/webroot - - run: - - sysrc nginx_enable=YES + - crontab -u root /root/crontab_root - - rm -v /root/crontab_root - - run: + - rm -vf /root/crontab_root + + - sysrc nginx_enable=YES - sysrc sshd_enable=NO - sysrc sendmail_enable=NONE - sysrc clear_tmp_enable=YES diff --git a/example/gateway/files/crontab_root b/example/gateway/nginx-https/files/crontab_root similarity index 100% rename from example/gateway/files/crontab_root rename to example/gateway/nginx-https/files/crontab_root diff --git a/focker/compose.py b/focker/compose.py index 5ccf12b..0175268 100644 --- a/focker/compose.py +++ b/focker/compose.py @@ -29,22 +29,32 @@ from .misc import focker_lock, \ import pdb -def exec_prebuild(spec, path): +def exec_hook(spec, path, hook_name='exec.prebuild'): if isinstance(spec, str): spec = [ spec ] if not isinstance(spec, list): - raise ValueError('exec.prebuild should be a string or a list of strings') + raise ValueError('%s should be a string or a list of strings' % hook_name) spec = ' && '.join(spec) - print('Running exec.build command:', spec) + print('Running %s command:' % hook_name, spec) spec = [ '/bin/sh', '-c', spec ] oldwd = os.getcwd() os.chdir(path) + focker_unlock() res = subprocess.run(spec) + focker_lock() if res.returncode != 0: - raise RuntimeError('exec.prebuild failed') + raise RuntimeError('%s failed' % hook_name) os.chdir(oldwd) +def exec_prebuild(spec, path): + return exec_hook(spec, path, 'exec.prebuild') + + +def exec_postbuild(spec, path): + return exec_hook(spec, path, 'exec.postbuild') + + def build_volumes(spec): poolname = zfs_poolname() for tag, params in spec.items(): @@ -120,8 +130,7 @@ def build_jails(spec): zfs_untag([ jailname ], focker_type='jail') zfs_tag(name, [ jailname ]) path = zfs_mountpoint(name) - generated_names[jailname] = \ - jail_create(path, + generated_names[jailname] = jail_create(path, jailspec.get('exec.start', '/bin/sh /etc/rc'), jailspec.get('env', {}), [ [from_, on] \ @@ -152,6 +161,8 @@ def command_compose_build(args): build_images(spec['images'], path, args) if 'jails' in spec: build_jails(spec['jails']) + if 'exec.postbuild' in spec: + exec_postbuild(spec['exec.postbuild'], path) def command_compose_run(args): diff --git a/focker/focker.py b/focker/focker.py index 140b56f..5b3562b 100644 --- a/focker/focker.py +++ b/focker/focker.py @@ -21,7 +21,8 @@ from .volume import command_volume_create, \ command_volume_prune, \ command_volume_list, \ command_volume_tag, \ - command_volume_untag + command_volume_untag, \ + command_volume_remove import sys from .zfs import zfs_init from .jail import command_jail_create, \ @@ -186,6 +187,11 @@ def create_parser(): parser.set_defaults(func=command_volume_untag) parser.add_argument('tags', type=str, nargs='+') + parser = ListForwarder([subparsers.add_parser(cmd) for cmd in ['remove', 'rm', 'r']]) + parser.set_defaults(func=command_volume_remove) + parser.add_argument('references', type=str, nargs='+') + parser.add_argument('--force', '-f', action='store_true') + # compose subparsers = ListForwarder([ subparsers_top.add_parser(cmd).add_subparsers(dest='L2_command') \ for cmd in ['compose', 'comp', 'c'] ]) diff --git a/focker/volume.py b/focker/volume.py index d270b57..5930325 100644 --- a/focker/volume.py +++ b/focker/volume.py @@ -44,3 +44,14 @@ def command_volume_tag(args): def command_volume_untag(args): zfs_untag(args.tags, focker_type='volume') + + +def command_volume_remove(args): + for ref in args.references: + try: + name, _ = zfs_find(ref, focker_type='volume') + print('Removing:', name) + zfs_run(['zfs', 'destroy', '-r', '-f', name]) + except: + if not args.force: + raise