#!/usr/local/bin/cbsd
#v12.0.6
MYARG="mode"
MYOPTARG="bootable display dsk_controller dsk_path dsk_sectorsize dsk_serial dsk_size header imgtype jname zvol_opts"
MYDESC="Manage bhyve ahci/virtio disk"
CBSDMODULE="bhyve"
EXTHELP="wf_bhyve"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

This command will show/manage virtual disks for bhyve VMs, e.g: add new disk, resize disks and so on.

When you use ZFS, you may want to adjust some properties (e.g. reservation/compression/..) by default
via ~cbsd/etc/zfs.conf ( defaults: ~cbsd/etc/defaults/zfs.conf )

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}mode=${N0_COLOR}          - action to be performed, options:
      - 'attach'             - create and attach new disk;
      - 'detach'             - detach disk from controller (without removing dsk);
      - 'delete' or 'remove' - detach and remove disk;
      - 'modify'             - modify disk properties (e.g: bootable=, dsk_sectorsize=,
                               dsk_serial=, dsk_size=, ..);
      - 'get_next_free_dsk'  - out next available dsk name for jname;
      - 'get'                - get properties;

 mode=modify properties:

 ${N2_COLOR}bootable${N0_COLOR}       - set bootable flag (1 - true, - false);
 ${N2_COLOR}dsk_sectorsize${N0_COLOR} - set sectorsize (e.g: 512, 4096, 512/4096);
 ${N2_COLOR}dsk_serial${N0_COLOR}     - set Serial Number with maximum 20 characters,
                  '0' for default/auto-generated value, see man bhyve_config(5);
 ${N2_COLOR}dsk_size${N0_COLOR}       - increase disk size (e.g: 20g, +30g );
 ${N2_COLOR}dsk_zfs_guid${N0_COLOR}   - can be: 'auto';

 mode=attach properties:

 ${N2_COLOR}zvol_opts${N0_COLOR}      - pass ZFS volume options, e.g:
                  zvol_opts=\"dedup=on sync=disabled\";

 mode=list properties:

  ${N2_COLOR}header=0${N0_COLOR}      - don't print header;

  ${N2_COLOR}display=${N0_COLOR}      - customize output, list item by comma, default values:
                 'jname,dsk_controller,dsk_path,dsk_size,dsk_sectorsize,bootable'
                  also available: 'dsk_zfs_guid,dsk_serial';

 ${N2_COLOR}imgtype${N0_COLOR}        - can be: 'zvol', 'md', 'raw';

${H3_COLOR}Examples${N0_COLOR}:

* Lets increase the size of the 'dsk1' for the 'freebsd2' virtual machine by '30g':

  # cbsd bhyve-dsk mode=modify dsk_size=+30g jname=freebsd2 dsk_controller=virtio-blk dsk_path=dsk1.vhd
  # cbsd bhyve-dsk mode=modify dsk_serial=\"BHYVE-1234567890\" jname=freebsd2 dsk_controller=virtio-blk dsk_path=dsk1.vhd
  # cbsd bhyve-dsk mode=attach jname=freebsd2 dsk_controller=virtio-blk dsk_size=10g
  # cbsd bhyve-dsk mode=attach jname=freebsd2 dsk_controller=virtio-blk dsk_size=10g imgtype=md
  # cbsd bhyve-dsk mode=delete jname=freebsd2 dsk_controller=virtio-blk dsk_path=dsk2
  # cbsd bhyve-dsk mode=attach jname=freebsd2 dsk_controller=virtio-blk dsk_path=/dev/da2 imgtype=raw

* List of registered disks:

  # cbsd bhyve-dsk mode=list display=jname,dsk_size,dsk_zfs_guid,dsk_serial

${H3_COLOR}See also${N0_COLOR}:

 cbsd media --help
 cbsd bconfig --help
 cat ~cbsd/etc/defaults/zfs.conf

"

. ${subrdir}/nc.subr
. ${system}

bootable=
dsk_iops_limit=
dsk_mbps_limit=
zvol_opts=
imgtype=
oimgtype=
. ${cbsdinit}

[ -n "${imgtype}" ] && oimgtype="${imgtype}"

available_properties="bootable dsk_sectorsize dsk_zfs_guid dsk_serial dsk_size"
available_get_properties="dsk_path bootable dsk_sectorsize dsk_serial dsk_zfs_guid"

. ${subrdir}/bhyve.subr
. ${subrdir}/virtual.subr
. ${distdir}/share/bhyve/bhyve-dsk.subr

dsk_attach()
{
	local _val _ret _zvol_opts _zvol_args _valid_ctrl_list _valid_ctrl _x
	local _res _raw=0

	if [ -z "${dsk_sectorsize}" ]; then
		readconf bhyve-default-default.conf
		dsk_sectorsize="${default_sectorsize}"
	fi

	[ -z "${dsk_serial}" ] && dsk_serial="0"

	# get next free disk
	if [ -z "${dsk_path}" ]; then
		dsk_path=$( get_next_free_dsk -j ${jname} -e bhyve )
		[ $? -ne 0 ] && dsk_path=
	fi

	for i in jname dsk_controller dsk_path dsk_size dsk_sectorsize; do
		_val=
		eval _val="\$$i"

		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				_res=$( substr --pos=0 --len=5 --str=${_val} )
				if [ "${_res}" = "/dev/" ]; then
					[ ! -c "${_val}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such character device: ${N2_COLOR}${_val}${N0_COLOR}"
					dsk_path="${_val}"
					imgtype="raw"
					_raw=1
				else
					dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				fi
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				[ -z "${jname}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}jname= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				_valid_ctrl=0
				_valid_ctrl_list=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite "SELECT name FROM bhyve_dskcontroller" 2>/dev/null | ${XARGS_CMD} )
				case "${_val}" in
					# add custom controller name
					virtio-blk|ahci-hd)
						;;
					*)
						for _x in ${_valid_ctrl_list}; do
							[ "${_val}" = "${_x}" ] && _valid_ctrl=1 && break
						done
						[ ${_valid_ctrl} -eq 0 ] && err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk ahci-hd ${_valid_ctrl_list}${N0_COLOR}"
						;;
				esac
				;;
		esac
	done


	# not mandatory for raw
	if [ ${_raw} -ne 1 ]; then
		for i in dsk_size dsk_sectorsize; do
			_val=
			eval _val="\$$i"

			[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
		done
	fi

	zvol_opts=

	if [ ${zfsfeat} -eq 1 ]; then
		[ -n "${oimgtype}" ] && imgtype="${oimgtype}"
		[ -z "${imgtype}" ] && imgtype="zvol"
	else
		# turn off zfsfeat
		zfsfeat=0
		[ -n "${oimgtype}" ] && imgtype="${oimgtype}"
		[ -z "${imgtype}" ] && imgtype="md"
	fi

	if [ ${zfsfeat} -eq 1 ]; then
		if [ -n "${zvol_opts}" ]; then
			for i in ${zvol_opts}; do
				if [ -z "${_zvol_opts}" ]; then
					_zvol_opts="-o ${i}"
				else
					_zvol_opts="${_zvol_opts} -o ${i}"
				fi
			done
		else
			_zvol_opts=
		fi
	fi

	if [ -n "${_zvol_opts}" ]; then
		add_dsk -c "${dsk_controller}" -d "${dsk_path}" -i "${imgtype}" -n "${dsk_serial}" -s "${dsk_size}" -o "${_zvol_opts}"  -z "${dsk_sectorsize}"
		_ret=$?
	else
		add_dsk -c "${dsk_controller}" -d "${dsk_path}" -i "${imgtype}" -n "${dsk_serial}" -s "${dsk_size}"  -z "${dsk_sectorsize}"
		_ret=$?
	fi

	if [ ${_ret} -eq 0 ]; then
		${ECHO} "${N2_COLOR}${dsk_path}${N1_COLOR} attached${N0_COLOR}"
	else
		err 1 "${N2_COLOR}${dsk_path}${N1_COLOR} attach failed${N0_COLOR}"
	fi

	return ${_ret}
}

dsk_detach()
{
	local _val _res
	local _devpath

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	_res=$( substr --pos=0 --len=5 --str=${dsk_path} )
	if [ "${_res}" = "/dev/" ]; then
		# raw device
		local lunname=$( ${BASENAME_CMD} ${dsk_path} )
		_devpath=$( cbsdsqlro storage_media SELECT path FROM media WHERE jname=\"${jname}\" AND type=\"hdd\" AND name=\"hdd-${lunname}\" LIMIT 1 2>/dev/null )
		[ -z "${_devpath}" ] && err 1 "${N1_COLOR}Unable to find in media DB: SELECT path FROM media WHERE jname=\"${jname}\" AND type=\"hdd\" AND name=\"hdd-${lunname}\"${N0_COLOR}"
	else
		# check in media table
		_devpath=$( cbsdsqlro storage_media SELECT path FROM media WHERE jname=\"${jname}\" AND type=\"hdd\" AND name=\"hdd-${dsk_path}.vhd\" LIMIT 1 2>/dev/null )
		[ -z "${_devpath}" ] && err 1 "${N1_COLOR}Unable to find in media DB: SELECT path FROM media WHERE jname=\"${jname}\" AND type=\"hdd\" AND name=\"hdd-${dsk_path}.vhd\"${N0_COLOR}"
	fi

	if [ "${_res}" = "/dev/" ]; then
		# raw device
		_val=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_path FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}\" AND dsk_type=\"vhd\" 2>/dev/null )
		[ -z "${_val}" ] && err 1 "${N1_COLOR}Unable to find in bhyvedsk: SELECT dsk_path FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}\" AND dsk_type=\"vhd\"${N0_COLOR}"
		media mode=detach name=hdd-${lunname} path=${_devpath} type=hdd jname=${jname}
	else
		_val=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_path FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" AND dsk_type=\"vhd\" 2>/dev/null )
		[ -z "${_val}" ] && err 1 "${N1_COLOR}Unable to find in bhyvedsk: SELECT dsk_path FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" AND dsk_type=\"vhd\"${N0_COLOR}"
		media mode=detach name=hdd-${dsk_path}.vhd path=${_devpath} type=hdd jname=${jname}
	fi

	if [ "${_res}" = "/dev/" ]; then
		# raw device
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite DELETE FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}\" AND dsk_type=\"vhd\"
		# scan for symlink to raw devices
		${FIND_CMD} ${jaildatadir}/${jname}-${jaildatapref}/ -mindepth 1 -maxdepth 1 -name dsk\*.vhd -type l | while read _link; do
			_realpath=$( ${READLINK_CMD} ${_link} )
			[ "${_realpath}" = "${dsk_path}" ] && ${RM_CMD} ${_link}
		done
	else
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite DELETE FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" AND dsk_type=\"vhd\"
	fi

	${ECHO} "${N2_COLOR}${dsk_path}${N1_COLOR} dettached (but not removed!)${N0_COLOR}"
}

dsk_delete()
{
	local _val _res
	local _devpath

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	_res=$( substr --pos=0 --len=5 --str=${dsk_path} )
	if [ "${_res}" = "/dev/" ]; then
		# raw device - detech only
		dsk_detach
		return 0
	fi

	# check in media table
	_devpath=$( cbsdsqlro storage_media SELECT path FROM media WHERE jname=\"${jname}\" AND type=\"hdd\" AND name=\"hdd-${dsk_path}.vhd\" LIMIT 1 2>/dev/null )
	if [ -z "${_devpath}" ]; then
		${ECHO} "${N1_COLOR}Unable to find in media DB: SELECT path FROM media WHERE jname=\"${jname}\" AND type=\"hdd\" AND name=\"hdd-${dsk_path}.vhd\"${N0_COLOR}" 2>&1
	fi

	_val=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_path FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" AND dsk_type=\"vhd\" 2>/dev/null )
	[ -z "${_val}" ] && err 1 "${N1_COLOR}Unable to find in bhyvedsk: SELECT dsk_path FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" AND dsk_type=\"vhd\"${N0_COLOR}"

	media mode=delete name=hdd-${dsk_path}.vhd path=${_devpath} type=hdd jname=${jname}

	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite DELETE FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" AND dsk_type=\"vhd\"
}

dsk_modify_bootable()
{
	local _bootable _res
	local _old_controller= _old_dsk=

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"

	case "${bootable}" in
		[Nn][Oo] | [Ff][Aa][Ll][Ss][Ee] | 0)
			_bootable="false"
			;;
		[Yy][Ee][Ss] | [Tt][Rr][Uu][Ee] | 1)
			_bootable="true"
			;;
		*)
			err 1 "${N1_COLOR}dsk_modify_bootable: unknown bootable: ${N2_COLOR}${bootable}. ${N1_COLOR}Valid: [1|true|yes] and [0|false|no]${N0_COLOR}"
			;;
	esac

	_res=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_controller,dsk_path FROM bhyvedsk WHERE bootable=${_bootable} LIMIT 1 | ${AWK_CMD} '{printf $1}' )
	sqllist "${_res}" _old_controller _old_dsk

	case "${_bootable}" in
		true)
			if [ "${_old_controller}" = "${dsk_controller}" ]; then
				if [ "${_old_dsk}" = "${dsk_path}" ]; then
					${ECHO} "${N1_COLOR}${jname}: ${dsk_controller} ${dsk_path} already bootable${N0_COLOR}"
					return 0
				fi
			fi

			# currently only one device can be bootable, so set false for all devices first
			cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE bhyvedsk SET bootable=false
			[ -n "${_res}" ] && ${ECHO} "${N1_COLOR}Update bootable flag for ${jname}:${_old_controller}:${_old_dsk} -> false${N0_COLOR}"
			${ECHO} "${N1_COLOR}Update bootable flag for ${jname}:${dsk_controller}:${dsk_path} -> ${N2_COLOR}${_bootable}${N0_COLOR}"
			;;
		false)
			if [ "${_old_controller}" = "${dsk_controller}" ]; then
				if [ "${_old_dsk}" = "${dsk_path}" ]; then
					${ECHO} "${N1_COLOR}${jname}: ${dsk_controller} ${dsk_path} already not bootable${N0_COLOR}"
					return 0
				fi
			fi
			${ECHO} "${N1_COLOR}Update bootable flag for ${jname}:${dsk_controller}:${dsk_path} -> ${N2_COLOR}${_bootable}${N0_COLOR}"
			;;
	esac

	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE bhyvedsk SET bootable=${_bootable} WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\"
	return 0
}

dsk_modify_dsk_sectorsize()
{
	local _res _old_val

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac
		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"

	_old_val=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_sectorsize FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" LIMIT 1 | ${AWK_CMD} '{printf $1}' )
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to get current dsk_sectorsize values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"
	[ "${_old_val}" = "${dsk_sectorsize}" ] && return 0
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE bhyvedsk SET dsk_sectorsize=\"${dsk_sectorsize}\" WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\"

	cbsdlogger NOTICE "${CBSD_APP}: modify sectorsize ${_old_val} -> ${dsk_sectorsize} (controller: ${dsk_controller}, dsk: ${dsk_path})"
	${ECHO} "${N1_COLOR}dsk_sectorsize: ${N2_COLOR}changed${N0_COLOR}"
	return 0
}

dsk_modify_dsk_serial()
{
	local _res _old_val

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac
		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"

	_old_val=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_serial FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" LIMIT 1 | ${AWK_CMD} '{printf $1}' )
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to get current dsk_serial values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"
	[ "${_old_val}" = "${dsk_serial}" ] && return 0
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE bhyvedsk SET dsk_serial=\"${dsk_serial}\" WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\"

	cbsdlogger NOTICE "${CBSD_APP}: modify sectorsize ${_old_val} -> ${dsk_serial} (controller: ${dsk_controller}, dsk: ${dsk_path})"
	${ECHO} "${N1_COLOR}dsk_serial: ${N2_COLOR}changed${N0_COLOR}"
	return 0
}

dsk_modify_dsk_zfs_guid()
{
	local _res _old_val tmp_zvol= _is_zvol
	local _dsk_fullpath _zvol_pref= _ret

	[ ${zfsfeat} -ne 1 ] && err 1 "${N1_COLOR}zfsfeat disabled${N0_COLOR}"
	[ "${dsk_zfs_guid}" != "auto" ] && err 1 "${N1_COLOR}dsk_zfs_guid valid values: 'auto'${N0_COLOR}"

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"

	_old_val=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_zfs_guid FROM bhyvedsk WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\" LIMIT 1 | ${AWK_CMD} '{printf $1}' )
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to get current dsk_zfs_guid values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"

	_dsk_fullpath="${jaildatadir}/${jname}-${jaildatapref}/${dsk_path}.vhd"

	. ${subrdir}/zfs.subr

	_res=$( get_dsk_zfs_guid -p ${_dsk_fullpath} 2>/dev/null )
	_ret=$?
	[ ${_ret} -ne 0 ] && err 0 "${N1_COLOR}skipp dsk_modify_dsk_zfs_guid error: unable to determine guid for: ${N2_COLOR}${_dsk_fullpath}${N0_COLOR}"
	[ -z "${_res}" ] && err 0 "${N1_COLOR}skipp dsk_modify_dsk_zfs_guid error: unable to determine guid for: ${N2_COLOR}${_is_zvol}${N0_COLOR}"
	[ "${_old_val}" = "${_res}" ] && return 0
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE bhyvedsk SET dsk_zfs_guid=\"${_res}\" WHERE dsk_controller=\"${dsk_controller}\" AND dsk_path=\"${dsk_path}.vhd\"
	cbsdlogger NOTICE "${CBSD_APP}: modify dsk_zfs_guid ${_old_val} -> ${dsk_zfs_guid} (controller: ${dsk_controller}, dsk: ${dsk_path})"
	${ECHO} "${N1_COLOR}dsk_zfs_guid: ${N2_COLOR}changed${N0_COLOR}"
	return 0
}

dsk_modify_dsk_size()
{
	local _res _old_val _new_val tmp_zvol= _is_zvol _inc_dsk_size _dsk_size
	local _dsk_fullpath _zvol_pref= _ret _dsk_size_pref _inc_dsk_size_bytes _new_dsk_size

	[ ${zfsfeat} -ne 1 ] && err 1 "${N1_COLOR}zfsfeat disabled${N0_COLOR}"

	for i in jname dsk_controller dsk_path dsk_size; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"

	# get dsk_size, dsk_bsize, dsk_realsize
	_dsk_size="${dsk_size}"
	populate_dsk_size ${jaildatadir}/${jname}-${jaildatapref}/${dsk_path}
	_old_val="${dsk_bsize}"
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to get current dsk_size values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"

	_dsk_size_pref=$( substr --pos=0 --len=1 --str="${_dsk_size}" )		# now we have dest param
	# increment ?
	if [ "${_dsk_size_pref}" = "+" ]; then
		# sure
		_inc_dsk_size=$( substr --pos=2 --len=0 --str="${_dsk_size}" )
		if is_number "${_inc_dsk_size}"; then
			if conv2bytes ${_inc_dsk_size}; then
				_inc_dsk_size_bytes="${convval}"
			else
				err 1 "${N1_COLOR}${CBSD_APP}: unable convert values to bytes: ${N2_COLOR}${_inc_dsk_size}${N0_COLOR}"
			fi
		else
			# already in bytes?
			_inc_dsk_size_bytes="${inc_dsk_size}"
		fi
		_new_val=$(( _old_val + _inc_dsk_size_bytes ))
		_new_dsk_size="${_new_val}"
	else
		_new_dsk_size="${_dsk_size}"		# store new value
		# fixed size
		if is_number "${_new_dsk_size}"; then
			if conv2bytes ${_new_dsk_size}; then
				_new_val="${convval}"
			else
				err 1 "${N1_COLOR}${CBSD_APP}: unable convert values to bytes: ${N2_COLOR}${_new_dsk_size}${N0_COLOR}"
			fi
		else
			# already in bytes?
			_new_val="${_new_dsk_size}"
		fi
	fi

	if [ ${_old_val} -gt ${_new_val} ]; then
		${ECHO} "${N1_COLOR}${CBSD_APP}: disk reduction is not yet supported${N0_COLOR}"
		err 1 "${N1_COLOR}new dsk_size must be larger than the old values ( >${N2_COLOR}${_old_val}${N1_COLOR} bytes )${N0_COLOR}"
	fi

	# check for available disksize/overcommiting ?
	_dsk_fullpath="${jaildatadir}/${jname}-${jaildatapref}/${dsk_path}.vhd"

	modify_dsk_size -f ${_dsk_fullpath} -s "${_new_val}"
	_ret=$?
	[ ${_ret} -ne 0 ] && return ${_ret}

	# update
	populate_dsk_size
	# update values in table
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET dsk_size='${dsk_bsize}' WHERE dsk_path='${dsk_path}.vhd'"
	${ECHO} "${N1_COLOR}dsk_size: ${N2_COLOR}changed${N0_COLOR}"
	return 0
}

dsk_modify()
{
	local _val
	local _devpath _prop_num=0

	# determine properties
	for i in ${available_properties}; do
		_val=
		eval _val="\$$i"
		[ -z "${_val}" ] && continue
		dsk_modify_${i}
		_prop_num=$(( _prop_num + 1 ))
	done

	[ ${_prop_num} -eq 0 ] && ${ECHO} "${N1_COLOR}available properties: ${N2_COLOR}${available_properties}${N0_COLOR}"
}

dsk_get()
{
	local _val
	local _devpath _prop_num=0
	local _filter= _mysql=

	# determine properties
	for i in ${available_get_properties}; do
		_val=
		eval _val="\$$i"

		if [ "${i}" = "dsk_path" ]; then
			# trim/cast .vhd if necessary
			_val=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
			_val="${_val}.vhd"
		fi

		[ -z "${_val}" ] && continue
		if [ -z "${_filter}" ]; then
			_filter="${i}=\"${_val}\""
		else
			_filter="${_filter} AND ${i}=\"${_val}\""
		fi
		_prop_num=$(( _prop_num + 1 ))
	done

	if [ ${_prop_num} -eq 0 ]; then
		${ECHO} "${N1_COLOR}set filter from available properties (select .. WHERE): ${N2_COLOR}${available_properties}${N0_COLOR}"
		return 1
	fi

	_mysql="SELECT ${get_value} FROM bhyvedsk WHERE ${_filter}"
	# echo "${_mysql}"
	cbsdsqlro ${jailsysdir}/${jname}/local.sqlite ${_mysql}
}

emulator="bhyve" # for jname_is_multiple
jname_is_multiple

if [ -n "${jail_list}" ]; then
	TMP_JLIST="${jail_list}"
else
	TMP_JLIST=$*
fi

JLIST=

# check for actual vm list in arg list
jail_num=0
for i in ${TMP_JLIST}; do
	exist=$( cbsdsqlro local SELECT jname FROM jails WHERE jname=\"${i}\" AND emulator=\"${emulator}\" LIMIT 1 )
	if [ -n "${exist}" ]; then
		JLIST="${exist} ${JLIST}"
		jail_num=$(( jail_num + 1 ))
	fi
done

# this is multiple list, split it by parallel bhyve-dsk execution
if [ ${jail_num} -gt 1 ]; then
	cbsdlogger NOTICE ${CBSD_APP}: executing for multiple bhyve-dsk: ${JLIST}
	new_arg=

	for i in $*; do
		_is_jname=$( substr --pos=0 --len=5 --str=${i} )
		[ "${_is_jname}" = "jname" ] && continue
		new_arg="${new_arg} ${i}"
	done

	for jname in ${JLIST}; do
		bhyve-dsk jname=${jname} ${new_arg}
	done
	exit 0
fi

if [ -n "${jname}" ]; then
	. ${subrdir}/rcconf.subr
	[ $? -eq 1 ] && err 1 "${N1_COLOR}bhyve-dsk: no such domain here: ${N2_COLOR}${jname}${N0_COLOR}"
	[ "${emulator}" != "bhyve" ] && err 1 "${N1_COLOR}bhyve-dsk: not in bhyve mode: ${N2_COLOR}${jname}${N0_COLOR}"
fi

case "${mode}" in
	attach)
		dsk_attach
		exit $?
		;;
	delete|remove)
		dsk_delete
		;;
	detach)
		dsk_detach
		;;
	list)
		[ -z "${display}" ] && display="jname,dsk_controller,dsk_path,dsk_size,dsk_sectorsize,bootable"
		[ -z "${header}" ] && header="1"
		if [ -n "${jname}" ]; then
			bhyve-dsk-list header="${header}" display="${display}" jname="${jname}"
		else
			bhyve-dsk-list header="${header}" display="${display}"
		fi
		;;
	modify)
		dsk_modify
		;;
	get)
		[ -z "${jname}" ] && err 1 "${N1_COLOR}jname= is mandatory${N0_COLOR}"
		. ${distsharedir}/bhyvedsk.conf
		# filter params=val and $argx value
		get_value=
		for i in $*; do
			# jname=XX dsk=YY test
			strpos --str="${i}" --search="="
			_pos=$?
			if [ ${_pos} -eq 0 ]; then
				if [ -z "${get_value}" ]; then
					get_value="${i}"
				else
					get_value="${get_value},${i}"
				fi
			fi
		done
		if [ -z "${get_value}" ]; then
			${ECHO} "${N1_COLOR}Empty value for get. Use: cbsd bhyve-dsk mode=get <filter> XXX, where XXX from:${N0_COLOR}"
			${ECHO} "   ${N2_COLOR}${MYCOL}${N0_COLOR}"
			exit 1
		fi
		dsk_get
		;;
	get_next_free_dsk)
		[ -z "${jname}" ] && err 1 "${N1_COLOR}jname= is mandatory${N0_COLOR}"
		next_dsk=$( get_next_free_dsk -j ${jname} -e bhyve )
		[ $? -ne 0 ] && return 1
		printf "${next_dsk}"
		return 0
		;;
	*)
		err 1 "${N1_COLOR}Unknown mode: ${N2_COLOR}${mode}${N0_COLOR}"
		;;
esac

exit 0
