#!/bin/sh # MapGen v3 installer -- https://mapgen.isaret.ai/install.sh # # curl -fsSL https://mapgen.isaret.ai/install.sh | sh # curl -fsSL https://mapgen.isaret.ai/install.sh | sh -s -- --channel rc # curl -fsSL https://mapgen.isaret.ai/install.sh | sh -s -- --uninstall # # Deploys the MapGen v3 Docker stack (redis, mapgend) on an Ubuntu 22.04 GPU # host: pulls the channel's prebuilt image, takes the docker-compose.yml that # ships inside it (so the file always matches the image), puts it and .env in # /opt/mapgen, starts the stack and waits for /api/v3/health. Rerun it to # update to the channel's latest image and compose file; .env is kept. A rerun # stops while a flight is running (the update restarts mapgend and would kill # it) unless --force is given. After that the stack is # plain Docker Compose: cd /opt/mapgen && sudo docker compose ps|logs|down. # The API has no authentication: keep its port on a trusted network. # # A site gets one key: a Docker Hub read-only access token and its account # name (the image is private). # # The host needs Docker Engine with the Compose plugin, the NVIDIA driver and # the NVIDIA Container Toolkit. The script checks them and stops with what to # install; it never installs them itself. # # Do not run this with sudo in front; it calls sudo itself where needed, and # `VAR=x sudo sh` drops the variables. The token is never put on a command # line: docker login reads it from stdin. # # POSIX sh. Nothing runs until main() at the very end, so a truncated download # executes nothing. Input is read from /dev/tty, never from stdin, because # stdin is the script itself when piped from curl. # # Environment (the two key values are asked for on the terminal when unset): # MAPGEN_CHANNEL stable (default) | rc | dev -- same as --channel # MAPGEN_DOCKER_USER Docker Hub account name (asked) # MAPGEN_DOCKER_TOKEN Docker Hub access token (asked) # MAPGEN_DIR default /opt/mapgen; its name is the Compose project # MAPGEN_DISK_ROOT parent of the flight drives' mount points, default # /mnt/mapgen (first install only) # MAPGEN_PORT host port of the API, default 1307 (first install only) # MAPGEN_BIND host address the API port is published on, default # 0.0.0.0 (every interface; first install only). Docker # publishes around ufw, so `ufw deny` does not close it set -eu IMAGE_REPO="docker.io/atlasimaging/mapgen_v3" # Where the mapgen-v3 package puts the production compose file in the image. COMPOSE_IN_IMAGE="/usr/share/mapgen-v3/docker-compose.yml" CHANNEL="${MAPGEN_CHANNEL:-stable}" DOCKER_USER="${MAPGEN_DOCKER_USER:-}" DOCKER_TOKEN="${MAPGEN_DOCKER_TOKEN:-}" DIR="${MAPGEN_DIR:-/opt/mapgen}" ACTION="install" FORCE="" SUDO="" IMAGE="" OLD_IMAGE_ID="" NEW_IMAGE_ID="" TMP="" say() { printf '%s\n' "$*" >&2; } die() { say "install.sh: error: $*"; exit 1; } usage() { cat </dev/null 2>&1 || die "run as root, or install sudo" SUDO="sudo" say "root is needed for docker and to write $DIR; sudo may ask for your password" $SUDO -v || die "sudo failed" fi } # Everything missing is reported with what to run. Nothing is installed here: # Docker and NVIDIA packages on a site host are a human call. check_host() { [ -r /etc/os-release ] || die "cannot read /etc/os-release; only Ubuntu 22.04 is supported" # shellcheck disable=SC1091 . /etc/os-release if [ "${ID:-}" != "ubuntu" ] || [ "${VERSION_CODENAME:-}" != "jammy" ]; then die "only Ubuntu 22.04 (jammy) is supported; this is ${PRETTY_NAME:-an unknown OS}" fi command -v curl >/dev/null 2>&1 || die "curl not found. Install with: sudo apt-get install -y curl ca-certificates" command -v docker >/dev/null 2>&1 || die "docker not found. Install Docker Engine and the Compose plugin (docker-ce, docker-compose-plugin): https://docs.docker.com/engine/install/ubuntu/" need_sudo $SUDO docker info >/dev/null 2>&1 || die "the Docker daemon does not answer (sudo docker info). Start it: sudo systemctl enable --now docker" $SUDO docker compose version >/dev/null 2>&1 || die "the Docker Compose plugin is missing. Install it: sudo apt-get install -y docker-compose-plugin (from Docker's apt repository)" _problems="" if ! command -v nvidia-smi >/dev/null 2>&1 || ! nvidia-smi >/dev/null 2>&1; then _problems="$_problems - NVIDIA driver (R550 or newer) not working: sudo ubuntu-drivers install, then reboot" fi if ! $SUDO docker info --format '{{json .Runtimes}}' 2>/dev/null | grep -q '"nvidia"'; then _problems="$_problems - Docker has no nvidia runtime: install the NVIDIA Container Toolkit (https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html), then: sudo nvidia-ctk runtime configure --runtime=docker && sudo systemctl restart docker" fi [ -z "$_problems" ] || die "the GPU stack is not ready:$_problems" } # /dev/tty is mode 0666 even without a controlling terminal, so test by # opening it, not by permission bits. have_tty() { (: /dev/null; } restore_echo() { stty echo /dev/null || true; } # ask VAR ENV_NAME "prompt" [hidden] ask() { have_tty || die "$2 is not set and there is no terminal to ask on" printf '%s: ' "$3" >/dev/tty if [ "${4:-}" = "hidden" ]; then trap restore_echo EXIT INT TERM stty -echo /dev/tty else read -r _answer /dev/null || die "docker login failed: wrong Docker Hub account name or token" OLD_IMAGE_ID=$($SUDO docker image inspect --format '{{.Id}}' "$IMAGE" 2>/dev/null || true) say "pulling $IMAGE" $SUDO docker pull "$IMAGE" || die "cannot pull $IMAGE: see docker's error above (a network problem: rerun; 'denied' or 'not found': the token has no read access to atlasimaging/mapgen_v3, or the $CHANNEL channel has no image yet)" NEW_IMAGE_ID=$($SUDO docker image inspect --format '{{.Id}}' "$IMAGE" 2>/dev/null || true) # The compose file is the one built into this image, not a copy from the # site, so a channel's file and image cannot drift apart. # Only a missing file means "predates"; any other docker failure (disk full, # daemon) is reported as what it is. if ! $SUDO docker run --rm --entrypoint cat "$IMAGE" "$COMPOSE_IN_IMAGE" >"$TMP" 2>"$_cat_err"; then if grep -q 'No such file' "$_cat_err"; then rm -f "$_cat_err" die "$IMAGE has no $COMPOSE_IN_IMAGE: that build predates the compose file shipping inside the image. Try another channel (--channel stable|rc|dev), or wait for this channel's next build" fi cat "$_cat_err" >&2 rm -f "$_cat_err" die "could not read $COMPOSE_IN_IMAGE from $IMAGE: see docker's error above" fi rm -f "$_cat_err" grep -q '^services:' "$TMP" || die "$COMPOSE_IN_IMAGE in $IMAGE is not a compose file" } # write_root_file PATH MODE -- content on stdin; created with a tight umask so # the redis password is never readable by others, not even before the chmod. write_root_file() { # shellcheck disable=SC2016 # $1/$2 are the inner sh's own positionals $SUDO sh -c 'umask 077 && cat >"$1" && chmod "$2" "$1"' sh "$1" "$2" } # The compose file is replaced on every run, with the one from the pulled # image. .env is written once; a rerun # changes only its MAPGEN_IMAGE line, so the password and hand edits stay. write_files() { $SUDO install -d -m 0755 "$DIR" write_root_file "$DIR/docker-compose.yml" 644 <"$TMP" if $SUDO test -f "$DIR/.env"; then say "keeping $DIR/.env; image is now $IMAGE" # shellcheck disable=SC2016 $SUDO sh -c 'if grep -q "^MAPGEN_IMAGE=" "$1"; then sed -i "s|^MAPGEN_IMAGE=.*|MAPGEN_IMAGE=$2|" "$1"; else printf "MAPGEN_IMAGE=%s\n" "$2" >>"$1"; fi' \ sh "$DIR/.env" "$IMAGE" else say "writing $DIR/.env (mode 600) with a generated redis password" { printf '# MapGen v3 site settings, read by docker-compose.yml. Every line also\n' printf '# reaches mapgend (add S3_*, WFD_DA2_MODEL, MAPGEN_AUTO_* here).\n' printf '# Auto (wf_datum depth) refuses a flight until the Depth Anything V2 ONNX model\n' printf '# is on the data volume and named here; no package ships it:\n' printf '# WFD_DA2_MODEL=/app/data/models/depth_anything_v2_vitl_dynamic_FIXED.onnx\n' printf '# After an edit: sudo docker compose up -d\n' printf 'MAPGEN_IMAGE=%s\n' "$IMAGE" printf 'REDIS_PASSWORD=%s\n' "$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')" printf 'MAPGEN_DISK_ROOT=%s\n' "${MAPGEN_DISK_ROOT:-/mnt/mapgen}" printf 'MAPGEN_PORT=%s\n' "${MAPGEN_PORT:-1307}" printf 'MAPGEN_BIND=%s\n' "${MAPGEN_BIND:-0.0.0.0}" } | write_root_file "$DIR/.env" 600 fi } compose() { $SUDO docker compose -f "$DIR/docker-compose.yml" "$@"; } # Sets _api to the installed stack's API, from the port and bind address in # $DIR/.env (a stack bound to one address is only reachable there). set_api() { _port=$($SUDO sed -n 's/^MAPGEN_PORT=//p' "$DIR/.env" | tail -n 1) _bind=$($SUDO sed -n 's/^MAPGEN_BIND=//p' "$DIR/.env" | tail -n 1) case "${_bind:-0.0.0.0}" in 0.0.0.0 | "") _bind="127.0.0.1" ;; esac _api="http://${_bind}:${_port:-1307}/api/v3" } # `compose up -d` recreates mapgend when the image or the compose file changed, # and that kills a flight in progress. On a rerun, ask the running daemon # (/health: gpuBusy, currentPipeline) and stop unless --force. A first install # or a stack that is really down has nothing to protect. A mapgend container # that is running but does not answer is not that: a daemon at 100 % CPU in the # middle of a flight can miss the 5 s timeout, so it stops too. guard_running_flight() { $SUDO test -f "$DIR/.env" || return 0 set_api # No pipe here: a pipeline's status is the last command's, which would hide # a curl failure. if _health=$(curl -fsS --max-time 5 "$_api/health" 2>/dev/null); then _health=$(printf '%s' "$_health" | tr -d ' \n\r') else $SUDO test -f "$DIR/docker-compose.yml" || return 0 [ -n "$(compose ps --status running -q mapgend 2>/dev/null)" ] || return 0 if [ -n "$FORCE" ]; then say "mapgend is running but does not answer $_api/health; --force: updating anyway, a flight in progress is killed" return 0 fi die "mapgend is running but does not answer $_api/health, so a flight in progress cannot be ruled out: updating restarts mapgend and would kill it. Look at it first (cd $DIR && sudo docker compose logs --tail 40 mapgend), then rerun; --force updates anyway" fi case "$_health" in *'"gpuBusy":true'*) ;; *) return 0 ;; esac _running=$(printf '%s' "$_health" | sed -n 's/.*"currentPipeline":"\([^"]*\)".*/\1/p') if [ -n "$FORCE" ]; then say "a flight is running (pipeline ${_running:-unknown}); --force: updating anyway, the flight is killed" return 0 fi die "a flight is running (pipeline ${_running:-unknown}): updating restarts mapgend and kills it. Wait for it, or cancel it (curl -X POST $_api/pipelines/${_running:-}/cancel), then rerun; --force updates anyway" } # Up to 5 min: mapgend's own start_period is 15 s. wait_healthy() { set_api _url="$_api/health" say "waiting for $_url" _i=0 while [ $_i -lt 60 ]; do if curl -fsS -o /dev/null "$_url" 2>/dev/null; then return 0 fi _i=$((_i + 1)) sleep 5 done compose ps >&2 || true compose logs --tail 40 mapgend >&2 || true die "no healthy answer from $_url after 5 min (logs above)" } # An update moves the channel's tag to the new image and leaves the old one on # disk, untagged. Remove that one image by id; it fails harmlessly while a # container still uses it or another tag points at it. prune_previous_image() { if [ -n "$OLD_IMAGE_ID" ] && [ -n "$NEW_IMAGE_ID" ] && [ "$OLD_IMAGE_ID" != "$NEW_IMAGE_ID" ]; then if $SUDO docker rmi "$OLD_IMAGE_ID" >/dev/null 2>&1; then say "removed the previous image" fi fi } do_install() { check_host guard_running_flight need_key say "" say "MapGen v3 install" say " channel : $CHANNEL" say " image : $IMAGE" say " into : $DIR" say "" fetch # The pull can take minutes, long enough for a flight to start since the # first check. This one is right before anything is replaced. guard_running_flight write_files compose pull --quiet redis compose up -d --remove-orphans wait_healthy prune_previous_image say "" say "MapGen v3 ($CHANNEL) is up." say " API : $_api, from other hosts through this one's address (no authentication: trusted network only;" say " Docker publishes around ufw, so 'ufw deny' does not close it: set MAPGEN_BIND in $DIR/.env)" say " manage : cd $DIR && sudo docker compose ps | logs -f mapgend | restart" say " settings : $DIR/.env, then: sudo docker compose up -d" say " Auto refuses flights until the Depth Anything V2 model is in place: set WFD_DA2_MODEL in $DIR/.env" say "update later: rerun this installer; switch channel: --channel {stable|rc|dev}; remove: --uninstall" say "a channel switch leaves the previous channel's image on disk: docker image ls atlasimaging/mapgen_v3" } # Stops the stack and removes $DIR. The data volumes (flight registry, redis) # stay, so a reinstall finds them. do_uninstall() { need_sudo if $SUDO test -f "$DIR/docker-compose.yml"; then say "stopping the stack" compose down || say "docker compose down failed; continuing" say "removing $DIR" $SUDO rm -f "$DIR/docker-compose.yml" "$DIR/.env" $SUDO rmdir "$DIR" 2>/dev/null || say "$DIR holds other files; left in place" else say "nothing installed in $DIR" fi say "done. Data volumes are kept (sudo docker volume ls); the Docker Hub login stays in root's ~/.docker/config.json (sudo docker logout)" } main() { parse_args "$@" case "$ACTION" in install) do_install ;; uninstall) do_uninstall ;; esac } main "$@"