===================================================================
RCS file: /cvs/openbsd/fw_update/fw_install.sh,v
retrieving revision 1.22
retrieving revision 1.122
diff -u -r1.22 -r1.122
--- openbsd/fw_update/fw_install.sh	2021/11/11 02:05:56	1.22
+++ openbsd/fw_update/fw_install.sh	2021/12/24 02:36:13	1.122
@@ -1,252 +1,431 @@
 #!/bin/ksh
-set -e
+#	$OpenBSD: fw_install.sh,v 1.122 2021/12/24 02:36:13 afresh1 Exp $
+#
+# Copyright (c) 2021 Andrew Hewus Fresh <afresh1@openbsd.org>
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-scan_dmesg() {
-	# no bsort for now
-	sed -n "$1" /var/run/dmesg.boot
-}
+set -o errexit -o pipefail -o nounset -o noclobber -o noglob
+set +o monitor
+export PATH=/usr/bin:/bin:/usr/sbin:/sbin
 
-prefetcharea_fs_list() {
-    echo "/mnt/tmp"
-}
-reset_watchdog() {
-}
+CFILE=SHA256.sig
+DESTDIR=${DESTDIR:-}
+FWPATTERNS="${DESTDIR}/usr/share/misc/firmware_patterns"
 
-# tmpdir, do_as, unpriv, and unpriv2 are from install.sub
+VNAME=${VNAME:-$(sysctl -n kern.osrelease)}
+VERSION=${VERSION:-"${VNAME%.*}${VNAME#*.}"}
 
-# Create a temporary directory based on the supplied directory name prefix.
+HTTP_FWDIR="$VNAME"
+VTYPE=$( sed -n "/^OpenBSD $VNAME\([^ ]*\).*$/s//\1/p" \
+    /var/run/dmesg.boot | sed '$!d' )
+[[ $VTYPE == -!(stable) ]] && HTTP_FWDIR=snapshots
+
+FWURL=http://firmware.openbsd.org/firmware/${HTTP_FWDIR}
+FWPUB_KEY=${DESTDIR}/etc/signify/openbsd-${VERSION}-fw.pub
+
+VERBOSE=false
+DELETE=false
+DOWNLOAD=true
+INSTALL=true
+LOCALSRC=
+
 tmpdir() {
 	local _i=1 _dir
 
-	until _dir="${1?}.$_i.$RANDOM" && mkdir -- "$_dir" 2>/dev/null; do
-		((++_i < 10000)) || return 1
-	done
+	# If we're not in the installer,
+	# we have mktemp and a more hostile environment.
+	if [ -x /usr/bin/mktemp ]; then
+		_dir=$( mktemp -d "${1}-XXXXXXXXX" )
+	else
+		until _dir="${1}.$_i.$RANDOM" && mkdir -- "$_dir" 2>/dev/null; do
+		    ((++_i < 10000)) || return 1
+		done
+	fi
+
 	echo "$_dir"
 }
 
-# Run a command ($2+) as unprivileged user ($1).
-# Take extra care that after "cmd" no "user" processes exist.
-#
-# Optionally:
-#	- create "file" and chown it to "user"
-#	- after "cmd", chown "file" back to root
-#
-# Usage: do_as user [-f file] cmd
-do_as() {
-	(( $# >= 2 )) || return
+fetch() {
+	local _src="${FWURL}/${1##*/}" _dst=$1 _user=_file _pid _exit _error=''
 
-	local _file _rc _user=$1
-	shift
+	# If we're not in the installer,
+	# we have su(1) and doas(1) is unlikely to be configured.
+	set -o monitor # make sure ftp gets its own process group
+	(
+	flags=-VM
+	"$VERBOSE" && flags=-vm
+	if [ -x /usr/bin/su ]; then
+		exec /usr/bin/su -s /bin/ksh "$_user" -c \
+		    "/usr/bin/ftp -N '${0##/}' -D 'Get/Verify' $flags -o- '$_src'" > "$_dst"
+	else
+		exec /usr/bin/doas -u "$_user" \
+		    /usr/bin/ftp -N "${0##/}" -D 'Get/Verify' $flags -o- "$_src" > "$_dst"
+	fi
+	) & _pid=$!
+	set +o monitor
 
-	if [[ $1 == -f ]]; then
-		_file=$2
-		shift 2
+	trap "kill -TERM '-$_pid' 2>/dev/null; exit 1" EXIT INT QUIT ABRT TERM
+
+	SECONDS=0
+	_last=0
+	while kill -0 -"$_pid" 2>/dev/null; do
+		if [[ $SECONDS -gt 12 ]]; then
+			set -- $( ls -ln "$_dst" 2>/dev/null )
+			if [[ $_last -ne $5 ]]; then
+				_last=$5
+				SECONDS=0
+				sleep 1
+			else
+				kill -INT -"$_pid"
+				_error=" (timed out)"
+			fi
+		else
+			sleep 1
+		fi
+	done
+
+	set +o errexit
+	wait "$_pid"
+	_exit=$?
+	set -o errexit
+
+	trap "" EXIT INT QUIT ABRT TERM
+
+	if [ "$_exit" -ne 0 ]; then
+		rm -f "$_dst"
+		echo "Cannot fetch $_src$_error" >&2
+		return 1
 	fi
+}
 
-	if [[ -n $_file ]]; then
-		>$_file
-		chown "$_user" "$_file"
+verify() {
+	# On the installer we don't get sha256 -C, so fake it.
+	if ! fgrep -qx "SHA256 (${1##*/}) = $( /bin/sha256 -qb "$1" )" "$CFILE"; then
+		echo "Checksum test for ${1##*/} failed." >&2
+		return 1
 	fi
+}
 
-	doas -u "$_user" "$@"
-	_rc=$?
+devices_needing_firmware() {
+	local _d _m _line _dmesgtail _last='' _nl=$( echo )
 
-	while doas -u "$_user" kill -9 -1 2>/dev/null; do
-		echo "Processes still running for user $_user after: $@"
-		sleep 1
-	done
+	# When we're not in the installer, the dmesg.boot can
+	# contain multiple boots, so only look in the last one
+	_dmesgtail="$( echo ; sed -n 'H;/^OpenBSD/h;${g;p;}' /var/run/dmesg.boot |
+	    grep -e "^[a-z][a-z]*[0-9]" -e " not configured " )"
 
-	[[ -n $_file ]] && chown root "$_file"
+	grep -v '^[[:space:]]*#' "$FWPATTERNS" |
+	    while read -r _d _m; do
+		[ "$_d" = "$_last" ] && continue
+		[ "$_m" ]             || _m="${_nl}${_d}[0-9] at "
+		[ "$_m" = "${_m#^}" ] || _m="${_nl}${_m#^}"
 
-	return $_rc
+		if [[ $_dmesgtail = *$_m* ]]; then
+			echo "$_d"
+			_last="$_d"
+		fi
+	    done
 }
 
-unpriv() {
-	do_as _sndio "$@"
+firmware_filename() {
+	local _f
+	_f="$( sed -n "s/.*(\($1-firmware-.*\.tgz\)).*/\1/p" "$CFILE" | sed '$!d' )"
+	! [ "$_f" ] && echo "Unable to find firmware for $1" >&2 && return 1
+	echo "$_f"
 }
 
-unpriv2() {
-	do_as _file "$@"
+firmware_devicename() {
+	local _d="${1##*/}"
+	_d="${_d%-firmware-*}"
+	echo "$_d"
 }
 
-# "fail" needs to be replaced with the "ask_yn" loop like in the installer.
-_issue=
-fail() {
-	echo $_issue >&2
-	exit 1
+installed_firmware() {
+	local _pre="$1" _match="$2" _post="$3" _firmware
+	set -A _firmware -- $(
+	    set +o noglob
+	    grep -Fxl '@option firmware' \
+		"${DESTDIR}/var/db/pkg/"$_pre"$_match"$_post"/+CONTENTS" \
+		2>/dev/null || true
+	    set -o noglob
+	)
+
+	[ "${_firmware[*]:-}" ] || return 0
+	for fw in "${_firmware[@]}"; do
+		fw="${fw%/+CONTENTS}"
+		echo "${fw##*/}"
+	done
 }
 
-VNAME=$(sysctl -n kern.osrelease)
-VERSION="${VNAME%.*}${VNAME#*.}"
-FWDIR="$VNAME"
+add_firmware () {
+	local _f="${1##*/}" _pkgname
+	local _tmpdir="$( tmpdir "${DESTDIR}/var/db/pkg/.firmware" )"
+	local flags=-VM
+	"$VERBOSE" && flags=-vm
+	ftp -N "${0##/}" -D "Install" "$flags" -o- "file:${1}" |
+		tar -s ",^\+,${_tmpdir}/+," \
+		    -s ",^firmware,${DESTDIR}/etc/firmware," \
+		    -C / -zxphf - "+*" "firmware/*"
 
-# TODO: We need the firmware for the system we just installed
-#       not the one we booted from.  For example:
-#       * booting from a snapshot bsd.rd that thinks it is the 7.0 release
-#         will install the firmware from the 7.0 directory instead of
-#         from the snapshots dir.
-#       If they're using sysupgrade, then the installer kernel will be correct.
-#       If we're doing this in the installer we can check what they picked
-#       for downloading sets and use that value.
-#       Otherwise, the fw_update after first boot will fix it up for us.
+	_pkgname="$( sed -n '/^@name /{s///p;q;}' "${_tmpdir}/+CONTENTS" )"
+	if [ ! "$_pkgname" ]; then
+		echo "Failed to extract name from $1, partial install" 2>&1
+		rm -rf "$_tmpdir"
+		return 1
+	fi
 
-HTTP_FWDIR=$FWDIR
-set -- $(scan_dmesg "/^OpenBSD $VNAME\([^ ]*\).*$/s//\1/p")
-[[ $1 == -!(stable) ]] && HTTP_FWDIR=snapshots
+	# TODO: Should we mark these so real fw_update can -Drepair?
+	ed -s "${_tmpdir}/+CONTENTS" <<EOL
+/^@comment pkgpath/ -1a
+@option manual-installation
+@option firmware
+@comment install-script
+.
+w
+EOL
 
-FWURL=http://firmware.openbsd.org/firmware/${HTTP_FWDIR}
-FWPUB_KEY=${DESTDIR}/etc/signify/openbsd-${VERSION}-fw.pub
-FWPATTERNS="file:${0%/*}/firmware_patterns"
+	chmod 755 "$_tmpdir"
+	mv "$_tmpdir" "${DESTDIR}/var/db/pkg/${_pkgname}"
+}
 
-# TODO: support srclocal installation of firmware somehow
-fw_update() {
-	local _src=$FWURL _tmpfs_list _tmpfs _tmpsrc
-    local _t=Get _cfile="/tmp/SHA256" _srclocal=false
-	local _f _r _remove _i _installed
-	local _pkgdir=/mnt/var/db/pkg
-	local _d _drivers=$(
-		last=''
-		ftp -D "Detecting" -Vmo- $FWPATTERNS |
-		while read _d _m; do
-			grep=grep
-			[ "$last" = "$_d" ] && continue
-			[ "$_m" ] || _m="^$_d[0-9][0-9]* at "
-			[ "$_m" = "${_m#^}" ] && grep=fgrep
-			$grep -q "$_m" /var/run/dmesg.boot || continue
-			echo $_d
-			last=$_d
-		done
-	)
+delete_firmware() {
+	local _cwd _pkg="$1" _pkgdir="${DESTDIR}/var/db/pkg"
 
-	if [ -z "$_drivers" ]; then
-		echo "No devices found which need firmware files to be downloaded."
-		return
+	# TODO: Check hash for files before deleting
+	"$VERBOSE" && echo "Uninstalling $_pkg"
+	_cwd="${_pkgdir}/$_pkg"
+
+	if [ ! -e "$_cwd/+CONTENTS" ] ||
+	    ! grep -Fxq '@option firmware' "$_cwd/+CONTENTS"; then
+		echo "${0##*/}: $_pkg does not appear to be firmware" >&2
+		return 2
 	fi
 
-	! _tmpfs_list=$(prefetcharea_fs_list) &&
-		echo "Cannot determine prefetch area" >&2 && return
+	set -A _remove -- "${_cwd}/+CONTENTS" "${_cwd}"
 
-	for _tmpfs in $_tmpfs_list; do
-		# Try to clean up from previous runs, assuming
-		# the _tmpfs selection yields the same mount
-		# point.
-		for _tmpsrc in $_tmpfs/firmware.+([0-9]).+([0-9]); do
-			[[ -d $_tmpsrc ]] && rm -r $_tmpsrc
-		done
+	while read -r c g; do
+		case $c in
+		@cwd) _cwd="${DESTDIR}$g"
+		  ;;
+		@*) continue
+		  ;;
+		*) set -A _remove -- "$_cwd/$c" "${_remove[@]}"
+		  ;;
+		esac
+	done < "${_pkgdir}/${_pkg}/+CONTENTS"
 
-		# Create a download directory for the firmware and
-		# check that the _sndio user can read files from
-		# it. Otherwise cleanup and skip the filesystem.
-		if _tmpsrc=$(tmpdir "$_tmpfs/firmware"); then
-			(
-			>$_tmpsrc/t &&
-			$_unpriv cat $_tmpsrc/t
-			) >/dev/null 2>&1 && break ||
-				rm -r $_tmpsrc
+	# We specifically rm -f here because not removing files/dirs
+	# is probably not worth failing over.
+	for _r in "${_remove[@]}" ; do
+		if [ -d "$_r" ]; then
+			# Try hard not to actually remove recursively
+			# without rmdir on the install media.
+			set +o noglob
+			[ "$_r/*" = "$( echo "$_r"/* )" ] && rm -rf "$_r"
+			set -o noglob
+		else
+			rm -f "$_r"
 		fi
 	done
+}
 
-	[[ ! -d $_tmpsrc ]] &&
-		echo "Cannot create prefetch area" >&2 && return 1
+usage() {
+	echo "usage:  ${0##*/} [-d | -D] [-av] [-p path] [driver | file ...]"
+	exit 2
+}
 
-	# Cleanup from previous runs.
-	rm -f $_cfile $_cfile.sig
+ALL=false
+OPT_D=
+while getopts :adDp:v name
+do
+       case "$name" in
+       a) ALL=true ;;
+       d) DELETE=true ;;
+       D) OPT_D=true ;;
+       p) LOCALSRC="$OPTARG" ;;
+       v) VERBOSE=true ;;
+       :)
+	   echo "${0##*/}: option requires an argument -- -$OPTARG" >&2
+	   usage 2
+	   ;;
+       ?)
+	   echo "${0##*/}: unknown option -- -$OPTARG" >&2
+	   usage 2
+	   ;;
+       esac
+done
+shift $((OPTIND - 1))
 
-	_t=Get/Verify
+if [ "$LOCALSRC" ]; then
+	if [[ $LOCALSRC = @(ftp|http?(s))://* ]]; then
+		FWURL="${LOCALSRC}"
+		LOCALSRC=
+	else
+		LOCALSRC="${LOCALSRC:#file:}"
+		! [ -d "$LOCALSRC" ] &&
+		    echo "The path must be a URL or an existing directory" >&2 &&
+		    exit 2
+	fi
+fi
 
-	! $_unpriv ftp -D "$_t" -Vmo - "$_src/SHA256.sig" >"$_cfile.sig" &&
-	    echo "Cannot fetch SHA256.sig" >&2 && return 1
+# "Download only" means local dir and don't install
+if [ "$OPT_D" ]; then
+	INSTALL=false
+	LOCALSRC="${LOCALSRC:-.}"
+fi
 
-	# Verify signature file with public keys.
-	! unpriv -f "$_cfile" \
-	    signify -Vep $FWPUB_KEY -x "$_cfile.sig" -m "$_cfile" &&
-	    echo "Signature check of SHA256.sig failed" >&2 && return 1
+[ "$LOCALSRC" ] || LOCALSRC="$( tmpdir "${DESTDIR}/tmp/${0##*/}" )"
 
-	for _d in $_drivers; do
-		$UU && reset_watchdog
-		_f=$( sed -n "s/.*(\($_d-firmware-.*\.tgz\)).*/\1/p" "$_cfile" )
-		_installed=$(
-	        for fw in "${_pkgdir}/$_d-firmware"*; do
-		        [ -e "$fw" ] || continue
-		        echo ${fw##*/}
-	        done
-		)
+CFILE="$LOCALSRC/$CFILE"
 
-		for _i in $_installed; do
-			if [ "$_f" = "$_i.tgz" ]; then
-				echo "Firmware for $_d already installed ($_installed)"
-				continue 2
+if [ -x /usr/bin/id ] && [ "$(/usr/bin/id -u)" != 0 ]; then
+	echo "need root privileges" >&2
+	exit 1
+fi
+
+set -A devices -- "$@"
+
+if "$DELETE"; then
+	[ "$OPT_D" ] && usage 22
+
+	set -A installed
+	if [ "${devices[*]:-}" ]; then
+		"$ALL" && usage 22
+
+		set -A installed -- $(
+		    for d in "${devices[@]}"; do
+			f="${d##*/}"  # only care about the name
+			f="${f%.tgz}" # allow specifying the package name
+			[ "$( firmware_devicename "$f" )" = "$f" ] && f="$f-firmware"
+
+			set -A i -- $( installed_firmware '' "$f-" '*' )
+
+			if [ "${i[*]:-}" ]; then
+				echo "${i[@]}"
+			else
+				echo "No firmware found for '$d'" >&2
 			fi
+		    done
+		)
+	elif "$ALL"; then
+		set -A installed -- $( installed_firmware '*' '-firmware-' '*' )
+	fi
+
+	deleted=''
+	if [ "${installed:-}" ]; then
+		for fw in "${installed[@]}"; do
+			delete_firmware "$fw" || continue
+			deleted="$deleted,$( firmware_devicename "$fw" )"
 		done
+	fi
 
-		rm -f /tmp/h /tmp/fail
+	deleted="${deleted:+${deleted#,}}"
+	echo "${0:##*/}: deleted ${deleted:-none}";
 
-		# Fetch firmware file and create a checksum by piping through
-		# sha256. Create a flag file in case ftp failed. Firmware
-		# from net is written to the prefetch area.
-		( $_unpriv ftp -D "$_t" -Vmo - "$_src/$_f" || >/tmp/fail ) |
-		( $_srclocal && unpriv2 sha256 -b >/tmp/h ||
-		    unpriv2 -f /tmp/h sha256 -bph /tmp/h >"$_tmpsrc/$_f" )
+	exit
+fi
 
-		# Handle failed transfer.
-		if [[ -f /tmp/fail ]]; then
-			rm -f "$_tmpsrc/$_f"
-			echo "Fetching of $_f failed!" >&2
-			continue
-		fi
+if [ "${devices[*]:-}" ]; then
+	"$ALL" && usage 22
+else
+	"$VERBOSE" && echo -n "Detecting firmware ..."
+	set -A devices -- $( devices_needing_firmware )
+	"$VERBOSE" &&
+	    { [ "${devices[*]:-}" ] && echo " found." || echo " done." ; }
+fi
 
-		# Verify firmware by comparing its checksum with SHA256.
-		if ! fgrep -qx "SHA256 ($_f) = $(</tmp/h)" "$_cfile"; then
-			[[ -d "$_tmpsrc" ]] && rm -rf "$_tmpsrc"
-			echo "Checksum test for $_f failed." >&2
-			continue
-		fi
+[ "${devices[*]:-}" ] || exit
 
-		# TODO: Check hash for files before deleting
-		if [ "$_installed" ] && [ -e "${_pkgdir}/$_installed/+CONTENTS" ]; then
-			echo "Uninstalling $_installed"
-			cwd=${_pkgdir}/$_installed
+if "$DOWNLOAD"; then
+	set +o noclobber # we want to get the latest CFILE
+	fetch "$CFILE"
+	set -o noclobber
+	! signify -qVep "$FWPUB_KEY" -x "$CFILE" -m "$CFILE" &&
+	    echo "Signature check of SHA256.sig failed" >&2 && exit 1
+elif [ ! -e "$CFILE" ]; then
+	# TODO: We shouldn't need a CFILE if all arguments are files.
+	echo "${0##*/}: $CFILE: No such file or directory" >&2
+	exit 2
+fi
 
-			set -A _remove -- "${cwd}/+CONTENTS" "${cwd}"
+added=''
+updated=''
+kept=''
+for f in "${devices[@]}"; do
+	d="$( firmware_devicename "$f" )"
 
-			while read c g; do
-				case $c in
-				@cwd) cwd="/mnt/$g"
-				  ;;
-				@*) continue
-				  ;;
-				*)  set -A _remove -- "$cwd/$c" "${_remove[@]}"
-				  ;;
-				esac
-			done < "${_pkgdir}/$_installed/+CONTENTS"
+	if [ "$f" = "$d" ]; then
+		f=$( firmware_filename "$d" || true )
+		[ "$f" ] || continue
+		f="$LOCALSRC/$f"
+	elif ! "$INSTALL" && ! grep -Fq "($f)" "$CFILE" ; then
+		echo "Cannot download local file $f" >&2
+		exit 2
+	fi
 
-			# We specifically rm -f here because not removing files/dirs
-			# is probably not worth failing over.
-			for _r in "${_remove[@]}" ; do
-				if [ -d "$_r" ]; then
-					# Try hard not to actually remove recursively
-					# without rmdir on the install media.
-					[ "$_r/*" = $( echo "$_r"/* ) ] && rm -rf "$_r"
-				else
-					rm -f "$_r"
-				fi
-			done
+	set -A installed -- $( installed_firmware '' "$d-firmware-" '*' )
+
+	if "$INSTALL" && [ "${installed[*]:-}" ]; then
+		for i in "${installed[@]}"; do
+			if [ "${f##*/}" = "$i.tgz" ]; then
+				"$VERBOSE" && echo "$i already installed"
+				kept="$kept,$d"
+				continue 2
+			fi
+		done
+	fi
+
+	if [ -e "$f" ]; then
+		if "$DOWNLOAD"; then
+			"$VERBOSE" && echo "Verify existing ${f##*/}"
+			verify "$f" || continue
+			"$INSTALL"  || kept="$kept,$d"
+		# else assume it was verified when downloaded
 		fi
+	elif "$DOWNLOAD"; then
+		fetch  "$f" || continue
+		verify "$f" || continue
+		"$INSTALL"  || added="$added,$d"
+	elif "$INSTALL"; then
+		echo "Cannot install ${f##*/}, not found" >&2
+		continue
+	fi
 
-		# TODO: Should we mark these so real fw_update can -Drepair?
-		echo "Installing $_f"
-		tar -zxphf "$_tmpsrc/$_f" -C "/mnt/etc" "firmware/*"
-		mkdir -p ${_pkgdir}/${_f%.tgz}/
-		tar -zxphf "$_tmpsrc/$_f" -C "${_pkgdir}/${_f%.tgz}" "+*"
-		ed -s "${_pkgdir}/${_f%.tgz}/+CONTENTS" <<EOL
-/^@comment pkgpath/ -1a
-@option manual-installation
-@option firmware
-@comment install-script
-.
-w
-EOL
-	done
-}
+	"$INSTALL" || continue
 
-fw_update
+	removed=false
+	if [ "${installed[*]:-}" ]; then
+		for i in "${installed[@]}"; do
+			delete_firmware "$i"
+			removed=true
+		done
+	fi
+
+	add_firmware "$f"
+
+	if "$removed"; then
+		updated="$updated,$d"
+	else
+		added="$added,$d"
+	fi
+done
+
+added="${added:#,}"
+updated="${updated:#,}"
+kept="${kept:#,}"
+if "$INSTALL"; then
+	echo  "${0##*/}: added ${added:-none}; updated ${updated:-none}; kept ${kept:-none}"
+else
+	echo  "${0##*/}: downloaded ${added:-none}; kept ${kept:-none}"
+fi