#!/usr/local/bin/cbsd
#v12.0.8
# calc_raw_data check_sha256 cloud conv2zvol convonly dstdir fastscan gen_sha256 keepname name path purge purgeonly
MYARG="dstdir"
MYOPTARG="calc_raw_data check_sha256 cloud conv2zvol convonly fastscan gen_sha256 keepname name path purge purgeonly"
MYDESC="Fetch ISO images from mirror sites"
CBSDMODULE="sys"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

 Script for downloading ISO/VM images according to profile templates ( ~cbsd/etc/defaults/vm-* + ~cbsd/etc/vm-*.conf )
 Mainly used in two scenarios.

1) Warming up images and creating a full mirror locally;
2) Check and update CRC SHA256 sum;

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}calc_raw_data=${N0_COLOR} - when 1, calculate and update imgsize_min for cloud image;
 ${N2_COLOR}check_sha256=${N0_COLOR}  - when 0, skip SHA256 CRC verification ( mutually exclusive for 'check_sha256' options,
                  if the sum does not match, the image will not be saved. default values is '1';
 ${N2_COLOR}cloud=${N0_COLOR}         - when 1, work with cloud image instead of iso ( 0 by default );
 ${N2_COLOR}conv2zvol=${N0_COLOR}     - when 1, try to convert ( keepname = 0 ) cloud images to ZVOL ( 0 by default );
 ${N2_COLOR}convonly=${N0_COLOR}      - convonly=/path/to/raw - apply conv2zvol to /path/to/raw only;
 ${N2_COLOR}dstdir=${N0_COLOR}        - destination directory. Use dstdir=default for using default iso dir: \$cbsd_workdir/src/iso;
 ${N2_COLOR}fastscan=${N0_COLOR}      - when 1, scan for fastest mirror first ( 0 by default );
 ${N2_COLOR}gen_sha256=${N0_COLOR}    - when 1: force to re-generate sha256sum files ( mutually exclusive for 'check_sha256' options,
                  default values is '0';
 ${N2_COLOR}keepname=${N0_COLOR}      - when 0, save ISO as register_iso_name variable, e.g cbsd-iso-${iso_img}
                  when '1', save ISO as original name on remote site (for mirroring);
 ${N2_COLOR}name=${N0_COLOR}          - fetch by profile name, e.g: FreeBSD-x64-11.1, Debian-x86-9;
 ${N2_COLOR}path=${N0_COLOR}          - fetch by profile path, e.g: /usr/jails/etc/defaults/vm-openbsd-x86-6.conf;
 ${N2_COLOR}purge=${N0_COLOR}         - when 1, purge ISO which are not referenced by any profiles;
 ${N2_COLOR}purgeonly=${N0_COLOR}     - when 1, no download, just purge ISO which are not referenced by any profiles;


${H3_COLOR}Examples${N0_COLOR}:

 1) Warm up ISO image for FreeBSD 13.0-RELEASE:

   a) by name:
   # cbsd fetch_iso name=FreeBSD-x64-13.0 dstdir=default fastscan=1

   b) by path:
   # cbsd fetch_iso path=/usr/jails/etc/defaults/vm-freebsd-FreeBSD-x64-13.0.conf dstdir=default fastscan=1

 2) Warm up FreeBSD 13.0-ZFS CLOUD image and convert to ZVOL as gold-image:

   # cbsd fetch_iso path="/usr/jails/etc/defaults/vm-freebsd-cloud-FreeBSD-zfs-x64-13.0.conf" conv2zvol=1 keepname=0 dstdir=default fastscan=1 cloud=1

 3) Warm up ALL images:

   # cbsd fetch_iso dstdir=default fastscan=1

 4) Create FULL mirror in /tmp/mirror with original images name ( profiles referenced name ):

   a) for ISO images:
   # mkdir /tmp/mirror/iso
   # cbsd fetch_iso dstdir=/tmp/mirror/iso keepname=1 purge=1 cloud=0

   b) for CLOUD images:
   # mkdir /tmp/mirror/cloud
   # cbsd fetch_iso dstdir=/tmp/mirror/cloud keepname=1 purge=1 cloud=1

${H3_COLOR}See also${N0_COLOR}:

 # cbsd media --help

 cat ~cbsd/etc/defaults/fetch_iso.conf - config file, use ~cbsd/etc/fetch_iso.conf to overwrite.
 url: https://github.com/cbsd/cbsd-vmprofiles

"

check_sha256=1
fastscan=0
calc_raw_data=0
cloud=0
conv2zvol=0
convonly=

. ${subrdir}/nc.subr

keepname=0
gen_sha256=0
name=
path=
purgeonly=0
purge=0
dstdir=

# summary size of processed ISO
processed_iso_size=0

# summary of bad ISO fetch
processed_iso_bad_count=0
processed_iso_bad_list=

# summary for purged ISOs
purged_iso_count=0
purged_iso_list=

readconf fetch_iso.conf

. ${cbsdinit}
. ${system}
. ${subrdir}/virtual.subr
. ${subrdir}/fetch.subr

[ "${dstdir}" = "default" ] && dstdir="${workdir}/src/iso"
[ ! -d ${dstdir} ] && ${MKDIR_CMD} -p ${dstdir}
if [ -n "${convonly}" ]; then
	[ ! -r "${convonly}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such file: ${N2_COLOR}${convonly}${N0_COLOR}"
	${ECHO} "${N1_COLOR}Trying to convert cloud images to zvol: ${N2_COLOR}${_source_raw}${N0_COLOR}"
	. ${subrdir}/zfs.subr
	convert_source_image_to_zvol -s "${convonly}"
	exit $?
fi

# gen_sha256
# $1 - path to profile which needs to be updated
# $2 - path to ISO
gen_sha256_sum()
{
	local _sha256sum
	local _profile="${1}"
	local _file="${2}"
	local _mdfile="${2}.sha256"
	local _iso_img_dist_size=0

	[ ! -r ${_profile} ] && return 0
	[ ! -r ${_file} ] && return 0

	printf "${N1_COLOR} calculating gen_sha256sum: ${N0_COLOR}"

	case "${platform}" in
		Linux)
			_sha256sum=$( ${NICE_CMD} -n 20 ${SHA256SUM_CMD} ${_file} | ${AWK_CMD} '{printf $1}' )
			;;
		*)
			_sha256sum=$( ${NICE_CMD} -n 20 ${SHA256_CMD} -q ${_file} )
			;;
	esac

	/usr/local/cbsd/misc/cbsdsysrc -qf ${_profile} sha256sum="${_sha256sum}"
	# update size

	case "${platform}" in
		Linux)
			_iso_img_dist_size=$( ${STAT_CMD} --printf="%s" ${_file} 2>/dev/null )
			;;
		*)
			_iso_img_dist_size=$( ${STAT_CMD} -f %z ${_file} 2>/dev/null )
			;;
	esac

	/usr/local/cbsd/misc/cbsdsysrc -qf ${_profile} iso_img_dist_size="${_iso_img_dist_size}"
	${ECHO} "${N2_COLOR}${_sha256sum}${N1_COLOR}. Updated${N0_COLOR}"
}

# must be: $dstdir, $fout
# $iso_site, $rfile
obtain_iso()
{
	if [ ${keepname} -eq 1 ]; then
		[ -r ${dstdir}/${fout} ] && return 0
	else
		# keepname 0
		[ -r ${dstdir}/${register_iso_name} ] && return 0
	fi

	if [ ${fastscan} -eq 1 ]; then
		scan_fastest_mirror -s "${iso_site_local} ${iso_site} ${cbsd_iso_mirrors}" -t 3 -u "${iso_img_dist}"
		iso_site="${FASTEST_SRC_MIRROR}"
	fi

	for site in ${iso_site}; do
		echo "fetch ${site}${rfile} -> ${dstdir}/${fout}"

		if [ -n "${vars_img}" ]; then
			if [ ! -s "${dstdir}/${vars_img}" ]; then
				${ECHO} "${N1_COLOR}Processing UEFI VARS file: ${N2_COLOR}${vars_img}${H2_COLOR}${N0_COLOR}"
				fetchme -o ${dstdir}/${vars_img} -u ${site}${vars_img}
			fi
		fi

		fetchme -o ${dstdir}/${fout} -u ${site}${rfile}

		if [ $? -ne 0 ]; then
			${RM_CMD} -f ${dstdir}/${fout}	# remove broken ISO
			continue
		fi

		if [ ${check_sha256} -eq 1 ]; then
			check_iso_sha256sum -f ${dstdir}/${fout} -m ${sha256sum}
			sha256sum_passed=$?
			if [ ${sha256sum_passed} -ne 1 ]; then
				${ECHO} "${N1_COLOR}SHA256 sum wrong. Please update profiles or use ${N2_COLOR}CBSD_ISO_SKIP_CHECKSUM=yes${N1_COLOR} variable" 1>&2
				${ECHO} "via env(1) or ${emulator}-default-default.conf config file to disable sha256sum validating${N0_COLOR}" 1>&2
				${RM_CMD} -f ${dstdir}/${fout}	# remove broken ISO
				continue
			fi
		fi

		if [ ${cloud} -eq 1 -a ${calc_raw_data} -eq 1 ]; then
			# stamp imgsize_min

			if [ -n "${iso_extract}" -a -f "${dstdir}/${fout}" ]; then
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Extracting ${dstdir}/${fout} -> ${dstdir}/${iso_img} ...${N0_COLOR}"
				set -o xtrace
				cd ${dstdir}
				/bin/sh -c "${iso_extract}"
				set +o xtrace

				case "${platform}" in
					Linux)
						_ebytes_raw_data=$( ${STAT_CMD} --printf="%s" ${dstdir}/${iso_img} 2>/dev/null )
						;;
					*)
						_ebytes_raw_data=$( ${STAT_CMD} -f "%z" ${dstdir}/${iso_img} 2>/dev/null )
						;;
				esac

				#_ebytes_raw_data=$(( _ebytes_raw_data * 512 ))					# real referenced size/data ( for: stat -f "%b" )
				${RM_CMD} -f ${dstdir}/${iso_img}
			else

				case "${platform}" in
					Linux)
						_ebytes_raw_data=$( ${STAT_CMD} --printf="%s" ${dstdir}/${fout} 2>/dev/null )
						;;
					*)
						_ebytes_raw_data=$( ${STAT_CMD} -f "%z" ${dstdir}/${fout} 2>/dev/null )
						;;
				esac

				#_ebytes_raw_data=$(( _ebytes_raw_data * 512 ))					# real referenced size/data ( for: stat -f "%b" )
			fi

			echo "/usr/local/cbsd/misc/cbsdsysrc -qf /usr/local/cbsd/etc/defaults/${profile_name} imgsize_min=${_ebytes_raw_data}"
			/usr/local/cbsd/misc/cbsdsysrc -qf /usr/local/cbsd/etc/defaults/${profile_name} imgsize_min="${_ebytes_raw_data}"
		fi

		if [ ${keepname} -eq 1 ]; then
			# no any action need
			return 0
		else
			if [ -n "${iso_extract}" -a -f "${dstdir}/${fout}" ]; then
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Extracting ${dstdir}/${fout} -> ${dstdir}/${iso_img} ...${N0_COLOR}"
				set -o xtrace
				cd ${dstdir}
				/bin/sh -c "${iso_extract}"
				set +o xtrace
				${MV_CMD} ${dstdir}/${iso_img} ${dstdir}/${register_iso_name}
			else
				# keepname 0
				${MV_CMD} ${dstdir}/${fout} ${dstdir}/${register_iso_name}
			fi
		fi
		return 0
	done

	return 1
}

get_active_profiles_num()
{
	profile_num=0

	for i in $( env NOCOLOR=1 show_profile_list show_cloud=${cloud} show_bhyve=1 search_profile=vm- display=path header=0 ); do

		[ ! -r ${i} ] && continue

		iso_site=
		iso_img=
		fetch=0
		sha256sum=0

		. ${i}

		[ ${fetch} -eq 0 ] && continue
		[ -z "${iso_site}"  ] && continue
		[ "${sha256sum}" = "0" ] && continue
		profile_num=$(( profile_num + 1 ))
	done

	printf "${profile_num}"
}

show_size_stats()
{
	local size=0

	if conv2human "${processed_iso_size}"; then
		size=${convval}
	else
		size="${processed_iso_size}"
	fi
	${ECHO} "${N1_COLOR}Processed ISO size: ${N2_COLOR}${size}${N0_COLOR}"
	if [ ${processed_iso_bad_count} -ne 0 ]; then
		${ECHO} "${N1_COLOR}Bad ISO count: ${W1_COLOR}${processed_iso_bad_count}${N0_COLOR}"
		${ECHO} "${N1_COLOR}Bad ISO list: ${W1_COLOR}${processed_iso_bad_list}${N0_COLOR}"
	fi
}

fetch_iso_by_path()
{
	local iso_bsize=0
	local _source_raw=
	# raw data size in bytes
	_ebytes_raw_data=0

	[ -n "${1}" ] && path="${1}"

	if [ -z "${path}" ]; then
		${ECHO} "${N1_COLOR}fetch_iso_by_path: empty path${N0_COLOR}"
		return 1
	fi

	# protect from long loop (2 - SIGINT in BSD)
	trap 'exit 0' 2

	iso_img=
	iso_img_dist=
	iso_site=
	fetch=0
	vm_profile=
	register_iso_name=
	sha256sum=0
	iso_extract=
	vars_img=

	if [ ! -r ${path} ]; then
		${ECHO} "${N1_COLOR}Unreadable profile: ${N2_COLOR}${path}${N0_COLOR}"
		return 1
	fi

	. ${path}

	if [ ${fetch} -eq 0 ]; then
		${ECHO} "${N1_COLOR}${path} fetch=0, skipp${N0_COLOR}"
		return 1
	fi

	if [ -z "${iso_site}"  ]; then
		${ECHO} "${N1_COLOR}${path} no iso_site, skipp${N0_COLOR}"
		return 1
	fi

	if [ -n "${iso_img_dist}" ]; then
		rfile="${iso_img_dist}"
	else
		rfile="${iso_img}"
	fi

	fout="${rfile}"

#	if [ ${keepname} -eq 1 ]; then
		fout="${rfile}"
#	else
#		fout="${register_iso_name}"
#	fi

	${ECHO} " ${H1_COLOR}* ${H5_COLOR}[${profile_cur}/${profile_num}] ${N2_COLOR}== ${N1_COLOR}Processing ${N2_COLOR}${vm_profile}${N1_COLOR} from ${N2_COLOR}${path}${N0_COLOR} =="

	if [ -z "${fout}" ]; then
		${ECHO} "fetch_iso_by_path: empty fout file"
		return 1
	fi

	profile_name=$( ${BASENAME_CMD} ${path} )

	obtain_iso
	ret=$?

	if [ ${ret} -ne 0 ]; then
		${ECHO} "${W1_COLOR}Warning: ${N1_COLOR}obtain_iso error for: ${H5_COLOR}${vm_profile}${N0_COLOR}"
		processed_iso_bad_count=$(( processed_iso_bad_count + 1 ))
		processed_iso_bad_list="${processed_iso_bad_list} ${vm_profile}"
	fi

	# check size
	case "${platform}" in
		Linux)
			iso_bsize=$( ${STAT_CMD} --printf="%s" ${dstdir}/${fout} 2>/dev/null )
			;;
		*)
			iso_bsize=$( ${STAT_CMD} -f "%z" ${dstdir}/${fout} 2>/dev/null )
			;;
	esac

	processed_iso_size=$(( processed_iso_size + iso_bsize ))

	# gen_sha256?
	if [ "${sha256sum}" != "0" ]; then
		gen_sha256_sum /usr/local/cbsd/etc/defaults/${profile_name} ${dstdir}/${fout}
	else
		${ECHO} "${N1_COLOR}${path} sha256sum=0, skipp${N0_COLOR}"
	fi
	echo

	if [ -r "${dstdir}/${register_iso_name}" ]; then
		_source_raw=${dstdir}/${register_iso_name}
	elif [ -r "${dstdir}/${iso_img}" ]; then
		_source_raw=${dstdir}/${iso_img}
	fi

	if [ ${cloud} -eq 1 -a ${conv2zvol} -eq 1 ]; then
		if [ "${keepname}" -eq 1 ]; then
			${ECHO} "${N1_COLOR}converting to ZFS volume disabled due to keepname=1. Please use keepname=0${N0_COLOR}"
			return 0
		fi
		if [ ! -r "${_source_raw}" ]; then
			${ECHO} "${N1_COLOR} no such raw file, skip convert to zvol: ${N2_COLOR}${_source_raw}${N0_COLOR}"
		else
			${ECHO} "${N1_COLOR}Trying to convert cloud images to zvol: ${N2_COLOR}${_source_raw}${N0_COLOR}"
			. ${subrdir}/zfs.subr
			convert_source_image_to_zvol -s "${_source_raw}"
		fi
	fi
}

# create array: (separated by spaces)
#   "profile_name|profile_path "
# in $profile_map variable
get_profile_name_path_map()
{
	# global
	profile_map=
	profile_num=0
	profile_cur=0

	profile_num=$( get_active_profiles_num )

	${ECHO} "${N1_COLOR}Active bhyve profiles here: ${N2_COLOR}${profile_num}${N0_COLOR}"

	# protect from long loop (2 - SIGINT)
	trap 'exit 0' 2

	for i in $( env NOCOLOR=1 show_profile_list show_cloud=${cloud} show_bhyve=1 search_profile=vm- display=path header=0 ); do
		iso_img=
		iso_img_dist=
		iso_site=
		fetch=0
		vm_profile=
		register_iso_name=
		sha256sum=0

		[ ! -r ${i} ] && continue
		. ${i}

		[ ${fetch} -eq 0 ] && continue
		[ -z "${iso_site}"  ] && continue
		[ "${sha256sum}" = "0" ] && continue
		[ -z "${vm_profile}" ] && continue
		profile_map="${profile_map}${vm_profile}|${i} "
		profile_cur=$(( profile_cur + 1 ))
	done
}

fetch_iso_all()
{
	profile_num=0
	profile_cur=0

	profile_num=$( get_active_profiles_num )

	${ECHO} "${N1_COLOR}Active bhyve profiles here: ${N2_COLOR}${profile_num}${N0_COLOR}"

	# protect from long loop (2 - SIGINT)
	trap 'exit 0' 2

	for i in $( env NOCOLOR=1 show_profile_list show_cloud=${cloud} show_bhyve=1 search_profile=vm- display=path header=0 ); do
		iso_img=
		iso_img_dist=
		iso_site=
		fetch=0
		vm_profile=
		register_iso_name=
		sha256sum=0
		iso_extract=

		[ ! -r ${i} ] && continue

		. ${i}

		[ ${fetch} -eq 0 ] && continue
		[ -z "${iso_site}"  ] && continue
		[ "${sha256sum}" = "0" ] && continue

		profile_cur=$(( profile_cur + 1 ))
		fetch_iso_by_path ${i}
	done
}


# ${active_iso_list} variable must be filled
exist_in_list()
{
	local _i

	[ -z "${active_iso_list}" ] && err 1 "${N1_COLOR}\$active_iso_list variables is empty${N0_COLOR}"
	[ -z "${1}" ] && err 1 "${N1_COLOR}exist_in_list <args>${N0_COLOR}"

	for _i in ${active_iso_list}; do
		[ "${_i}" = "${1}" ] && return 0
	done

	return 1
}

purge_iso_all()
{
	# global, used by fetch_iso_by_path func
	profile_num=0
	profile_cur=1

	profile_num=$( get_active_profiles_num )

	${ECHO} "${N1_COLOR}Active bhyve profiles here: ${N2_COLOR}${profile_num}${N0_COLOR}"
	${ECHO} "${N1_COLOR}Scan dir for purge/obsolete ISO's: ${N2_COLOR}${dstdir}${N0_COLOR}"

	active_iso_list=

	for i in $( env NOCOLOR=1 show_profile_list show_cloud=${cloud} show_bhyve=1 search_profile=vm- display=path header=0 ); do
		iso_img=
		iso_img_dist=
		iso_site=
		fetch=0
		vm_profile=
		register_iso_name=
		sha256sum=0
		iso_extract=

		[ ! -r ${i} ] && continue

		. ${i}

		[ ${fetch} -eq 0 ] && continue
		[ -z "${iso_site}"  ] && continue
		[ "${sha256sum}" = "0" ] && continue

		profile_cur=$(( profile_cur + 1 ))

		if [ -n "${iso_img_dist}" ]; then
			rfile="${iso_img_dist}"
		else
			rfile="${iso_img}"
		fi

		if [ ${keepname} -eq 1 ]; then
			fout="${rfile}"
		else
			fout="${register_iso_name}"
		fi

		if [ -z "${fout}" ]; then
			echo "Empty fout file"
			continue
		else
			printf " ${H1_COLOR}* ${H5_COLOR}[${profile_cur}/${profile_num}] ${N1_COLOR}Active ISO: ${N2_COLOR}${fout}${N1_COLOR} from ${N2_COLOR}${i}: ${N0_COLOR}"
		fi

		active_iso_list="${active_iso_list} ${fout}"

		if [ -r "${dstdir}/${fout}" ]; then
			${ECHO} "${H3_COLOR}Found${N0_COLOR}"
		else
			${ECHO} "${N1_COLOR}Not Found${N0_COLOR}"
		fi
	done

	if [ -z "${active_iso_list}" ]; then
		${ECHO} "${N1_COLOR}No active iso here${N0_COLOR}"
		return 0
	fi

	current_iso_list=$( ${FIND_CMD} ${dstdir}/ -mindepth 1 -maxdepth 1 -type f -exec ${BASENAME_CMD} {} \; | ${XARGS_CMD} );

	purged_iso_count=0

	for i in ${current_iso_list}; do
		if exist_in_list ${i}; then
			continue
		else
			${ECHO} "${N1_COLOR}Prune for: ${H5_COLOR}${i}${N0_COLOR}"
			${RM_CMD} -f ${dstdir}/${i}
			purged_iso_count=$(( purged_iso_count + 1 ))
			purged_iso_list="${purged_iso_list} ${i}"
		fi
	done

	${ECHO} "${N1_COLOR}Purged: ${N2_COLOR}${purged_iso_count}${N0_COLOR}"
	[ ${purged_iso_count} -gt 0 ] && ${ECHO} "${N1_COLOR}Purged ISO list: ${W1_COLOR}${purged_iso_list}${N0_COLOR}"
}

[ -n "${name}" -a -n "${path}" ] && err 1 "${N1_COLOR}Please use ${N2_COLOR}name= ${N1_COLOR}OR ${N2_COLOR}path=${N0_COLOR}"

if [ ${purgeonly} -eq 1 ]; then
	purge_iso_all
	exit 0
fi

if [ -z "${name}" -a -z "${path}" ]; then
	fetch_iso_all
	[ ${purge} -eq 1 ] && purge_iso_all
	show_size_stats
	exit 0
fi

if [ -n "${path}" ]; then
	# global, used by fetch_iso_by_path func
	profile_num=1
	profile_cur=1

	fetch_iso_by_path ${path}
	show_size_stats
	exit 0
fi

get_profile_path_by_name()
{
	[ -n "${1}" ] && name="${1}"

	if [ -z "${name}" ]; then
		${ECHO} "${N1_COLOR}get_profile_path_by_name: empty name${N0_COLOR}"
		return 1
	fi

	local p1 p2

	for i in ${profile_map}; do
		p1=${i%%|*}
		p2=${i##*|}
		if [ "${p1}" = "${name}" ]; then
			printf "${p2}"
			return 0
		fi
	done
}

if [ -n "${name}" ]; then
	get_profile_name_path_map
	profile_path=$( get_profile_path_by_name ${name} )
	if [ -n "${profile_path}" ]; then
		${ECHO} "${N1_COLOR}Profile name ${N2_COLOR}${name}${N1_COLOR} belongs to ${N2_COLOR}${profile_path}${N1_COLOR} profile.${N0_COLOR}"
		# global, used by fetch_iso_by_path func
		profile_num=1
		profile_cur=1
		fetch_iso_by_path ${profile_path}
	else
		${ECHO} "${N1_COLOR}No profile found with name: ${N2_COLOR}${name}${N0_COLOR}"
	fi
	show_size_stats
	exit 0
fi
