#!/usr/local/bin/cbsd
#v12.0.6
CIXARG="mode"
CIXOPTARG="bootable display dsk_controller dsk_path dsk_sectorsize dsk_serial dsk_size header imgtype jname zvol_opts"
MYDESC="Manage bhyve ahci/virtio disk"
CBSDMODULE="bhyve"
EXTHELP="wf_bhyve"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

This command will show/manage virtual disks for bhyve VMs, e.g: add new disk, resize disks and so on.

When you use ZFS, you may want to adjust some properties (e.g. reservation/compression/..) by default
via ~cbsd/etc/zfs.conf ( defaults: ~cbsd/etc/defaults/zfs.conf )

Please note when adding disks: by default you add each disk to a separate virtual PCI slot 
(the number of which is limited). If you are creating a virtual machine with more than 30 disks, 
please use virtual controllers ( bconfig -> bhyve_controller )

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}mode=${N0_COLOR}          - action to be performed, options:
      - 'attach'             - create and attach new disk;
      - 'detach'             - detach disk from controller (without removing dsk);
      - 'delete' or 'remove' - detach and remove disk;
      - 'modify'             - modify disk properties (e.g: bootable=, dsk_sectorsize=,
                               dsk_serial=, dsk_size=, ..);
      - 'get_next_free_dsk'  - out next available dsk name for jname;
      - 'get'                - get properties;

 mode=modify properties:

 ${N2_COLOR}bootable${N0_COLOR}       - set bootable flag (1 - true, - false);
 ${N2_COLOR}dsk_sectorsize${N0_COLOR} - set sectorsize (e.g: 512, 4096, 512/4096);
 ${N2_COLOR}dsk_serial${N0_COLOR}     - set Serial Number with maximum 20 characters,
                  '0' for default/auto-generated value, see man bhyve_config(5);
 ${N2_COLOR}dsk_size${N0_COLOR}       - increase disk size (e.g: 20g, +30g );
 ${N2_COLOR}dsk_zfs_guid${N0_COLOR}   - can be: 'auto';

 mode=attach properties:

 ${N2_COLOR}zvol_opts${N0_COLOR}      - pass ZFS volume options, e.g:
                  zvol_opts=\"dedup=on sync=disabled\";

 mode=list properties:

  ${N2_COLOR}header=0${N0_COLOR}      - don't print header;

  ${N2_COLOR}display=${N0_COLOR}      - customize output, list item by comma, default values:
                 'jname,dsk_controller,dsk_path,dsk_size,dsk_sectorsize,bootable'
                  also available: 'dsk_zfs_guid,dsk_serial';

 ${N2_COLOR}imgtype${N0_COLOR}        - can be: 'zvol', 'md', 'raw';

${H3_COLOR}Examples${N0_COLOR}:

* Lets increase the size of the 'dsk1' for the 'freebsd2' virtual machine by '30g':

  # cbsd bhyve-dsk mode=modify dsk_size=+30g jname=freebsd2 dsk_controller=virtio-blk dsk_path=dsk1.vhd
  # cbsd bhyve-dsk mode=modify dsk_serial=\"BHYVE-1234567890\" jname=freebsd2 dsk_controller=virtio-blk dsk_path=dsk1.vhd
  # cbsd bhyve-dsk mode=attach jname=freebsd2 dsk_controller=virtio-blk dsk_size=10g
  # cbsd bhyve-dsk mode=attach jname=freebsd2 dsk_controller=virtio-blk dsk_size=10g imgtype=md
  # cbsd bhyve-dsk mode=delete jname=freebsd2 dsk_controller=virtio-blk dsk_path=dsk2
  # cbsd bhyve-dsk mode=attach jname=freebsd2 dsk_controller=virtio-blk dsk_path=/dev/da2 imgtype=raw

* List of registered disks:

  # cbsd bhyve-dsk mode=list display=jname,dsk_size,dsk_zfs_guid,dsk_serial

${H3_COLOR}See also${N0_COLOR}:

 cbsd media --help
 cbsd bconfig --help
 bhyve-controller --help
 bhyve-controller-tui --help
 bhyve-nvme --help
 bhyve-nvme-tui --help
 cat ~cbsd/etc/defaults/zfs.conf

"

. ${subrdir}/nc.subr
. ${system}
. ${tools}

bootable=
dsk_iops_limit=
dsk_mbps_limit=
zvol_opts=
imgtype=
oimgtype=
cixinit

[ -n "${imgtype}" ] && oimgtype="${imgtype}"

available_properties="bootable dsk_sectorsize dsk_zfs_guid dsk_serial dsk_size"
available_get_properties="dsk_path bootable dsk_sectorsize dsk_serial dsk_zfs_guid"

. ${subrdir}/bhyve.subr
. ${subrdir}/virtual.subr
. ${CIX_DISTDIR}/share/bhyve/bhyve-dsk.subr

dsk_attach()
{
	local _val= _ret= _zvol_opts= _zvol_args= _valid_ctrl_list= _valid_ctrl= _x=
	local _res= _raw=0 _y= _first=

	if [ -z "${dsk_sectorsize}" ]; then
		readconf bhyve-default-default.conf
		dsk_sectorsize="${default_sectorsize}"
	fi

	[ -z "${dsk_serial}" ] && dsk_serial="0"

	# get next free disk
	if [ -z "${dsk_path}" ]; then
		capture dsk_path get_next_free_dsk -j ${jname} -e bhyve
		[ $? -ne 0 ] && dsk_path=
	fi

	for i in jname dsk_controller dsk_path dsk_size dsk_sectorsize; do
		_val=
		eval _val="\$$i"

		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				capture _res substr --pos=0 --len=5 --str="${_val}"
				if [ "${_res}" = "/dev/" ]; then
					[ ! -c "${_val}" ] && err 1 "${N1_COLOR}${CIX_APP}: no such character device: ${N2_COLOR}${_val}${N0_COLOR}"
					dsk_path="${_val}"
					imgtype="raw"
					_raw=1
				else
					#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
					dsk_path="${_val%.vhd}"
				fi
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				[ -z "${jname}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}jname= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				_valid_ctrl=0
				#_valid_ctrl_list=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite "SELECT name FROM bhyve_dskcontroller" 2>/dev/null | ${XARGS_CMD} )

				cbsdsqlro_vars ${jailsysdir}/${jname}/local.sqlite "SELECT name FROM bhyve_dskcontroller ORDER BY name ASC" _res
				_first=1
				_valid_ctrl_list=$( for _y in ${_res}; do
					[ ${_first} -eq 0 ] && printf " "
					_first=0
					printf "${_i}"
				done )

				case "${_val}" in
					# add custom controller name
					virtio-blk|ahci-hd|nvme)
						true
						;;
					*)
						for _x in ${_valid_ctrl_list}; do
							[ "${_val}" = "${_x}" ] && _valid_ctrl=1 && break
						done
						[ ${_valid_ctrl} -eq 0 ] && err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk ahci-hd nvme ${_valid_ctrl_list}${N0_COLOR}"
						;;
				esac
				;;
		esac
	done

	# not mandatory for raw
	if [ ${_raw} -ne 1 ]; then
		for i in dsk_size dsk_sectorsize; do
			_val=
			eval _val="\$$i"
			[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
		done
	fi

	zvol_opts=

	if [ ${zfsfeat} -eq 1 ]; then
		[ -n "${oimgtype}" ] && imgtype="${oimgtype}"
		[ -z "${imgtype}" ] && imgtype="zvol"
	else
		# turn off zfsfeat
		zfsfeat=0
		[ -n "${oimgtype}" ] && imgtype="${oimgtype}"
		[ -z "${imgtype}" ] && imgtype="md"
	fi

	if [ ${zfsfeat} -eq 1 -a "${imgtype}" = "zvol" ]; then
		if [ -n "${zvol_opts}" ]; then
			for i in ${zvol_opts}; do
				if [ -z "${_zvol_opts}" ]; then
					_zvol_opts="-o ${i}"
				else
					_zvol_opts="${_zvol_opts} -o ${i}"
				fi
			done
		else
			_zvol_opts=
		fi
	fi

	if [ -n "${_zvol_opts}" ]; then
		add_dsk -c "${dsk_controller}" -d "${dsk_path}" -i "${imgtype}" -n "${dsk_serial}" -s "${dsk_size}" -o "${_zvol_opts}"  -z "${dsk_sectorsize}"
		_ret=$?
	else
		add_dsk -c "${dsk_controller}" -d "${dsk_path}" -i "${imgtype}" -n "${dsk_serial}" -s "${dsk_size}"  -z "${dsk_sectorsize}"
		_ret=$?
	fi

	if [ ${_ret} -eq 0 ]; then
		${ECHO} "${N2_COLOR}${dsk_path}${N1_COLOR} attached${N0_COLOR}"
	else
		err 1 "${N2_COLOR}${dsk_path}${N1_COLOR} attach failed${N0_COLOR}"
	fi

	return ${_ret}
}

dsk_detach()
{
	local _val= _res= _devpath= lunname=

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				if [ -z "${_val}" ]; then
					${ECHO} "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
					bhyve-dsk mode=list jname=${jname} display=dsk_controller,dsk_path
					exit 1
				fi
				# trim .vhd if necessary
				#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				dsk_path="${_val%.vhd}"
				;;
			dsk_controller)
				if [ -z "${_val}" ]; then
					${ECHO} "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
					bhyve-dsk mode=list jname=${jname} display=dsk_controller,dsk_path
					exit 1
				fi
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						true
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	capture _res substr --pos=0 --len=5 --str="${dsk_path}"
	if [ "${_res}" = "/dev/" ]; then
		# raw device
		# basename
		lunname="${{dsk_path##*/}"
		cbsdsqlro_vars storage_media "SELECT path FROM media WHERE jname='${jname}' AND type='hdd' AND ( name='hdd-${lunname}' OR name='hdd-${lunname}.vhd' ) LIMIT 1" _devpath
		[ -z "${_devpath}" ] && err 1 "${N1_COLOR}Unable to find in media DB: SELECT path FROM media WHERE jname='${jname}' AND type='hdd' AND ( name='hdd-${lunname}' OR name='hdd-${lunname}.vhd' );${N0_COLOR}"
	else
		# check in media table
		cbsdsqlro_vars storage_media "SELECT path FROM media WHERE jname='${jname}' AND type='hdd' AND ( name='hdd-${dsk_path}.vhd' OR name='hdd-${dsk_path}' ) LIMIT 1" _devpath
		[ -z "${_devpath}" ] && err 1 "${N1_COLOR}Unable to find in media DB: SELECT path FROM media WHERE jname='${jname}' AND type='hdd' AND ( name='hdd-${dsk_path}.vhd' OR name='hdd-${dsk_path}' );${N0_COLOR}"
	fi

	if [ "${_res}" = "/dev/" ]; then
		# raw device
		cbsdsqlro_vars "${jailsysdir}/${jname}/local.sqlite" "SELECT dsk_path FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND ( dsk_path='${dsk_path}' OR dsk_path='${dsk_path}.vhd' ) AND dsk_type='vhd' LIMIT 1" _val
		[ -z "${_val}" ] && err 1 "${N1_COLOR}Unable to find in bhyvedsk: SELECT dsk_path FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND ( dsk_path='${dsk_path}' OR dsk_path='${dsk_path}.vhd' ) AND dsk_type='vhd'${N0_COLOR}"
		media mode=detach name=hdd-${lunname} path=${_devpath} type=hdd jname=${jname}
	else
		cbsdsqlro_vars "${jailsysdir}/${jname}/local.sqlite" "SELECT dsk_path FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND ( name='hdd-${dsk_path}.vhd' OR name='hdd-${dsk_path}' ) AND dsk_type='vhd' LIMIT 1" _val
		[ -z "${_val}" ] && err 1 "${N1_COLOR}Unable to find in bhyvedsk: SELECT dsk_path FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND ( name='hdd-${dsk_path}.vhd' OR name='hdd-${dsk_path}' ) AND dsk_type='vhd'${N0_COLOR}"
		media mode=detach name=hdd-${dsk_path}.vhd path=${_devpath} type=hdd jname=${jname}
	fi

	if [ "${_res}" = "/dev/" ]; then
		# raw device
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "DELETE FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}' AND dsk_type='vhd'"
		# scan for symlink to raw devices
		${FIND_CMD} ${jaildatadir}/${jname}-${jaildatapref}/ -mindepth 1 -maxdepth 1 -name dsk\*.vhd -type l | while read _link; do
			capture _realpath ${READLINK_CMD} "${_link}"
			[ "${_realpath}" = "${dsk_path}" ] && ${RM_CMD} ${_link}
		done
	else
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "DELETE FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' AND dsk_type='vhd'"
	fi

	${ECHO} "${N2_COLOR}${dsk_path}${N1_COLOR} dettached (but not removed!)${N0_COLOR}"
}

dsk_delete()
{
	local _val= _res= _devpath=

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				if [ -z "${_val}" ]; then
					${ECHO} "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
					bhyve-dsk mode=list jname=${jname} display=dsk_controller,dsk_path
					exit 1
				fi
				# trim .vhd if necessary
				#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				dsk_path="${_val%.vhd}"
				;;
			dsk_controller)
				if [ -z "${_val}" ]; then
					${ECHO} "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
					bhyve-dsk mode=list jname=${jname} display=dsk_controller,dsk_path
					exit 1
				fi
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						true
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	capture _res substr --pos=0 --len=5 --str="${dsk_path}"
	if [ "${_res}" = "/dev/" ]; then
		# raw device - detech only
		dsk_detach
		return 0
	fi

	# check in media table

	cbsdsqlro_vars storage_media "SELECT path FROM media WHERE jname='${jname}' AND type='hdd' AND name='hdd-${dsk_path}.vhd' LIMIT 1" _devpath
	if [ -z "${_devpath}" ]; then
		${ECHO} "${N1_COLOR}Unable to find in media DB: SELECT path FROM media WHERE jname='${jname}' AND type='hdd' AND name='hdd-${dsk_path}.vhd'${N0_COLOR}" 2>&1
	fi

	cbsdsqlro_vars ${jailsysdir}/${jname}/local.sqlite "SELECT dsk_path FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' AND dsk_type='vhd'" _val
	[ -z "${_val}" ] && err 1 "${N1_COLOR}Unable to find in bhyvedsk: SELECT dsk_path FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' AND dsk_type='vhd'${N0_COLOR}"

	media mode=delete name=hdd-${dsk_path}.vhd path=${_devpath} type=hdd jname=${jname}

	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "DELETE FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' AND dsk_type='vhd'"
}

dsk_modify_bootable()
{
	local _bootable= _res= _old_controller= _old_dsk=

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				dsk_path="${_val%.vhd}"
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						true
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac
		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"

	case "${bootable}" in
		[Nn][Oo] | [Ff][Aa][Ll][Ss][Ee] | 0)
			_bootable="false"
			;;
		[Yy][Ee][Ss] | [Tt][Rr][Uu][Ee] | 1)
			_bootable="true"
			;;
		*)
			err 1 "${N1_COLOR}dsk_modify_bootable: unknown bootable: ${N2_COLOR}${bootable}. ${N1_COLOR}Valid: [1|true|yes] and [0|false|no]${N0_COLOR}"
			;;
	esac

	. ${strings}

	cbsdsqlro_vars ${jailsysdir}/${jname}/local.sqlite "SELECT dsk_controller,dsk_path FROM bhyvedsk WHERE bootable='${_bootable}' LIMIT 1" _old_controller _old_dsk

	case "${_bootable}" in
		true)
			if [ "${_old_controller}" = "${dsk_controller}" ]; then
				if [ "${_old_dsk}" = "${dsk_path}" ]; then
					${ECHO} "${N1_COLOR}${jname}: ${dsk_controller} ${dsk_path} already bootable${N0_COLOR}"
					return 0
				fi
			fi

			# currently only one device can be bootable, so set false for all devices first
			cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET bootable=false"
			[ -n "${_res}" ] && ${ECHO} "${N1_COLOR}Update bootable flag for ${jname}:${_old_controller}:${_old_dsk} -> false${N0_COLOR}"
			${ECHO} "${N1_COLOR}Update bootable flag for ${jname}:${dsk_controller}:${dsk_path} -> ${N2_COLOR}${_bootable}${N0_COLOR}"
			;;
		false)
			if [ "${_old_controller}" = "${dsk_controller}" ]; then
				if [ "${_old_dsk}" = "${dsk_path}" ]; then
					${ECHO} "${N1_COLOR}${jname}: ${dsk_controller} ${dsk_path} already not bootable${N0_COLOR}"
					return 0
				fi
			fi
			${ECHO} "${N1_COLOR}Update bootable flag for ${jname}:${dsk_controller}:${dsk_path} -> ${N2_COLOR}${_bootable}${N0_COLOR}"
			;;
	esac

	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET bootable=${_bootable} WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd'"
	return 0
}

dsk_modify_dsk_sectorsize()
{
	local _res= _old_val=

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				dsk_path="${_val%.vhd}"
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						true
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac
		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"
	cbsdsqlro_vars ${jailsysdir}/${jname}/local.sqlite "SELECT dsk_sectorsize FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' LIMIT 1" _old_val
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CIX_APP}: unable to get current dsk_sectorsize values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"
	[ "${_old_val}" = "${dsk_sectorsize}" ] && return 0
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET dsk_sectorsize='${dsk_sectorsize}' WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd'"
	cbsdlogger NOTICE "${CIX_APP}: modify sectorsize ${_old_val} -> ${dsk_sectorsize} (controller: ${dsk_controller}, dsk: ${dsk_path})"
	${ECHO} "${N1_COLOR}dsk_sectorsize: ${N2_COLOR}changed${N0_COLOR}"

	return 0
}

dsk_modify_dsk_serial()
{
	local _res= _old_val=

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				dsk_path="${_val%.vhd}"
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						true
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac
		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"
	cbsdsqlro_vars ${jailsysdir}/${jname}/local.sqlite "SELECT dsk_serial FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' LIMIT 1" _old_val
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CIX_APP}: unable to get current dsk_serial values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"
	[ "${_old_val}" = "${dsk_serial}" ] && return 0
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET dsk_serial='${dsk_serial}' WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd'"
	cbsdlogger NOTICE "${CIX_APP}: modify sectorsize ${_old_val} -> ${dsk_serial} (controller: ${dsk_controller}, dsk: ${dsk_path})"
	${ECHO} "${N1_COLOR}dsk_serial: ${N2_COLOR}changed${N0_COLOR}"

	return 0
}

dsk_modify_dsk_zfs_guid()
{
	local _res= _old_val= tmp_zvol= _is_zvol=
	local _dsk_fullpath= _zvol_pref= _ret=

	[ ${zfsfeat} -ne 1 ] && err 1 "${N1_COLOR}zfsfeat disabled${N0_COLOR}"
	[ "${dsk_zfs_guid}" != "auto" ] && err 1 "${N1_COLOR}dsk_zfs_guid valid values: 'auto'${N0_COLOR}"

	for i in jname dsk_controller dsk_path; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				dsk_path="${_val%.vhd}"
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						true
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"
	#_old_val=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite "SELECT dsk_zfs_guid FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' LIMIT 1" | ${AWK_CMD} '{printf $1}' )
	cbsdsqlro_vars ${jailsysdir}/${jname}/local.sqlite "SELECT dsk_zfs_guid FROM bhyvedsk WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd' LIMIT 1" _old_val
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CIX_APP}: unable to get current dsk_zfs_guid values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"
	_dsk_fullpath="${jaildatadir}/${jname}-${jaildatapref}/${dsk_path}.vhd"
	. ${subrdir}/zfs.subr
	capture _res get_dsk_zfs_guid -p ${_dsk_fullpath} 2>/dev/null
	_ret=$?
	[ ${_ret} -ne 0 ] && err 0 "${N1_COLOR}skipp dsk_modify_dsk_zfs_guid error: unable to determine guid for: ${N2_COLOR}${_dsk_fullpath}${N0_COLOR}"
	[ -z "${_res}" ] && err 0 "${N1_COLOR}skipp dsk_modify_dsk_zfs_guid error: unable to determine guid for: ${N2_COLOR}${_is_zvol}${N0_COLOR}"
	[ "${_old_val}" = "${_res}" ] && return 0
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET dsk_zfs_guid='${_res}' WHERE dsk_controller='${dsk_controller}' AND dsk_path='${dsk_path}.vhd'"
	cbsdlogger NOTICE "${CIX_APP}: modify dsk_zfs_guid ${_old_val} -> ${dsk_zfs_guid} (controller: ${dsk_controller}, dsk: ${dsk_path})"
	${ECHO} "${N1_COLOR}dsk_zfs_guid: ${N2_COLOR}changed${N0_COLOR}"
	return 0
}

dsk_modify_dsk_size()
{
	local _res _old_val _new_val tmp_zvol= _is_zvol _inc_dsk_size _dsk_size
	local _dsk_fullpath _zvol_pref= _ret _dsk_size_pref _inc_dsk_size_bytes _new_dsk_size

	[ ${zfsfeat} -ne 1 ] && err 1 "${N1_COLOR}zfsfeat disabled${N0_COLOR}"

	for i in jname dsk_controller dsk_path dsk_size; do
		_val=
		eval _val="\$$i"
		case "${i}" in
			dsk_path)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# trim .vhd if necessary
				#dsk_path=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
				dsk_path="${_val%.vhd}"
				;;
			dsk_controller)
				[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
				# check for valid controller
				case "${_val}" in
					virtio-blk|ahci-hd)
						true
						;;
					*)
						err 1 "${N1_COLOR}bhyve-dsk error: bhyve-dsk: unknown dsk_controller ${_val}, valid: ${N2_COLOR}virtio-blk, ahci-hd${N0_COLOR}"
						;;
				esac
				;;
		esac

		[ -z "${_val}" ] && err 1 "${N1_COLOR}bhyve-dsk error: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
	done

	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && err 1 "${N1_COLOR}No such database: ${N2_COLOR}${jailsysdir}/${jname}/local.sqlite${N0_COLOR}"

	# get dsk_size, dsk_bsize, dsk_realsize
	_dsk_size="${dsk_size}"
	populate_dsk_size ${jaildatadir}/${jname}-${jaildatapref}/${dsk_path}
	_old_val="${dsk_bsize}"
	[ -z "${_old_val}" ] && err 1 "${N1_COLOR}${CIX_APP}: unable to get current dsk_size values for: dsk_controller=${N2_COLOR}${dsk_controller}${N1_COLOR} AND dsk_path=${N2_COLOR}${dsk_path}${N0_COLOR}"

	# now we have dest param
	capture _dsk_size_pref substr --pos=0 --len=1 --str="${_dsk_size}"
	# increment ?
	if [ "${_dsk_size_pref}" = "+" ]; then
		# sure
		capture _inc_dsk_size substr --pos=2 --len=0 --str="${_dsk_size}"
		if is_number "${_inc_dsk_size}"; then
			if conv2bytes ${_inc_dsk_size}; then
				_inc_dsk_size_bytes="${convval}"
			else
				err 1 "${N1_COLOR}${CIX_APP}: unable convert values to bytes: ${N2_COLOR}${_inc_dsk_size}${N0_COLOR}"
			fi
		else
			# already in bytes?
			_inc_dsk_size_bytes="${inc_dsk_size}"
		fi
		_new_val=$(( _old_val + _inc_dsk_size_bytes ))
		_new_dsk_size="${_new_val}"
	else
		_new_dsk_size="${_dsk_size}"		# store new value
		# fixed size
		if is_number "${_new_dsk_size}"; then
			if conv2bytes ${_new_dsk_size}; then
				_new_val="${convval}"
			else
				err 1 "${N1_COLOR}${CIX_APP}: unable convert values to bytes: ${N2_COLOR}${_new_dsk_size}${N0_COLOR}"
			fi
		else
			# already in bytes?
			_new_val="${_new_dsk_size}"
		fi
	fi

	if [ ${_old_val} -gt ${_new_val} ]; then
		${ECHO} "${N1_COLOR}${CIX_APP}: disk reduction is not yet supported${N0_COLOR}"
		err 1 "${N1_COLOR}new dsk_size must be larger than the old values ( >${N2_COLOR}${_old_val}${N1_COLOR} bytes )${N0_COLOR}"
	fi

	# check for available disksize/overcommiting ?
	_dsk_fullpath="${jaildatadir}/${jname}-${jaildatapref}/${dsk_path}.vhd"

	modify_dsk_size -f ${_dsk_fullpath} -s "${_new_val}"
	_ret=$?
	[ ${_ret} -ne 0 ] && return ${_ret}

	# update
	populate_dsk_size
	# update values in table
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET dsk_size='${dsk_bsize}' WHERE dsk_path='${dsk_path}.vhd'"
	${ECHO} "${N1_COLOR}dsk_size: ${N2_COLOR}changed${N0_COLOR}"

	return 0
}

dsk_modify()
{
	local _val= _devpath= _prop_num=0

	# determine properties
	for i in ${available_properties}; do
		_val=
		eval _val="\$$i"
		[ -z "${_val}" ] && continue
		dsk_modify_${i}
		_prop_num=$(( _prop_num + 1 ))
	done

	[ ${_prop_num} -eq 0 ] && ${ECHO} "${N1_COLOR}available properties: ${N2_COLOR}${available_properties}${N0_COLOR}"
}

dsk_get()
{
	local _val= _res= _devpath= _prop_num=0  _filter= _mysql=

	# determine properties
	for i in ${available_get_properties}; do
		_val=
		eval _val="\$$i"

		if [ "${i}" = "dsk_path" ]; then
			# trim/cast .vhd if necessary
			#_val=$( echo ${_val} | ${SED_CMD} 's:\.vhd::g' )
			_res="${_val%.vhd}"
			_val="${_res}.vhd"
		fi

		[ -z "${_val}" ] && continue
		if [ -z "${_filter}" ]; then
			_filter="${i}='${_val}'"
		else
			_filter="${_filter} AND ${i}='${_val}'"
		fi
		_prop_num=$(( _prop_num + 1 ))
	done

	if [ ${_prop_num} -eq 0 ]; then
		${ECHO} "${N1_COLOR}set filter from available properties (select .. WHERE): ${N2_COLOR}${available_properties}${N0_COLOR}"
		return 1
	fi

	_mysql="SELECT ${get_value} FROM bhyvedsk WHERE ${_filter}"
	# echo "${_mysql}"
	cbsdsqlro ${jailsysdir}/${jname}/local.sqlite ${_mysql}
}

emulator="bhyve" # for jname_is_multiple
jname_is_multiple

if [ -n "${jail_list}" ]; then
	TMP_JLIST="${jail_list}"
else
	TMP_JLIST=$*
fi

JLIST=

# check for actual vm list in arg list
jail_num=0
for i in ${TMP_JLIST}; do
	exist=$( cbsdsqlro local "SELECT jname FROM jails WHERE jname='${i}' AND emulator='${emulator}' LIMIT 1" )
	if [ -n "${exist}" ]; then
		JLIST="${exist} ${JLIST}"
		jail_num=$(( jail_num + 1 ))
	fi
done

# this is multiple list, split it by parallel bhyve-dsk execution
if [ ${jail_num} -gt 1 ]; then
	cbsdlogger NOTICE ${CIX_APP}: executing for multiple bhyve-dsk: ${JLIST}
	new_arg=

	for i in ${CIX_OTHER_ARGS}; do
		new_arg="${new_arg} ${i}"
	done

	for jname in ${JLIST}; do
		bhyve-dsk jname=${jname} ${new_arg}
	done
	exit 0
fi

if [ -n "${jname}" ]; then
	. ${subrdir}/rcconf.subr
	[ $? -eq 1 ] && err 1 "${N1_COLOR}bhyve-dsk: no such domain here: ${N2_COLOR}${jname}${N0_COLOR}"
	[ "${emulator}" != "bhyve" ] && err 1 "${N1_COLOR}bhyve-dsk: not in bhyve mode: ${N2_COLOR}${jname}${N0_COLOR}"
fi

case "${mode}" in
	attach)
		dsk_attach
		exit $?
		;;
	delete|remove)
		dsk_delete
		;;
	detach)
		dsk_detach
		;;
	list)
		[ -z "${display}" ] && display="jname,dsk_controller,dsk_path,dsk_size,dsk_sectorsize,bootable"
		[ -z "${header}" ] && header="1"
		if [ -n "${jname}" ]; then
			bhyve-dsk-list header="${header}" display="${display}" jname="${jname}"
		else
			bhyve-dsk-list header="${header}" display="${display}"
		fi
		;;
	modify)
		dsk_modify
		;;
	get)
		[ -z "${jname}" ] && err 1 "${N1_COLOR}jname= is mandatory${N0_COLOR}"
		. ${distsharedir}/bhyvedsk.conf
		# filter params=val and $argx value
		get_value=
		for i in $*; do
			# jname=XX dsk=YY test
			strpos --str="${i}" --search="="
			_pos=$?
			if [ ${_pos} -eq 0 ]; then
				if [ -z "${get_value}" ]; then
					get_value="${i}"
				else
					get_value="${get_value},${i}"
				fi
			fi
		done
		if [ -z "${get_value}" ]; then
			${ECHO} "${N1_COLOR}Empty value for get. Use: cbsd bhyve-dsk mode=get <filter> XXX, where XXX from:${N0_COLOR}"
			${ECHO} "   ${N2_COLOR}${MYCOL}${N0_COLOR}"
			exit 1
		fi
		dsk_get
		;;
	get_next_free_dsk)
		[ -z "${jname}" ] && err 1 "${N1_COLOR}jname= is mandatory${N0_COLOR}"
		capture next_dsk get_next_free_dsk -j ${jname} -e bhyve
		[ $? -ne 0 ] && return 1
		printf "${next_dsk}"
		return 0
		;;
	*)
		err 1 "${N1_COLOR}Unknown mode: ${N2_COLOR}${mode}${N0_COLOR}"
		;;
esac

exit 0
