#!/usr/local/bin/cbsd
#v12.0.5
MYARG=""
MYOPTARG="cfg_only debug debug_engine delay inter jname lm quiet"
MYDESC="Start QEMU domain"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

Start the qemu domain. You can start the offline machine, or continue the work
of the frozen machine from the checkpoint.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}cfg_only=${N0_COLOR}         - when set (e.g:'1'), find/create tap/vnc and
                     generate qemu args but without run, e.g for debugging.
 ${N2_COLOR}debug=${N0_COLOR}            - more debug messages.
 ${N2_COLOR}debug_engine=${N0_COLOR}     - overwrite debug_engine settings: use gdb or lldb
                     as debugger when launch qemu ( mostly for inherits debug
                     with live migration ).
 ${N2_COLOR}delay=${N0_COLOR}            - <sec>, delay N secbefore start, mainly to smooth
                     the astart, default is: '0', no delay.
 ${N2_COLOR}inter=${N0_COLOR}            - set 1 to prevent any questions and to accept answers by default.
 ${N2_COLOR}jname=${N0_COLOR}            - Target VM. If jail='*' or jail='vm*' then start  all
                     qemu or VM whose names begin with 'vm', e.g. 'vm1', 'vm2'...
 ${N2_COLOR}lm=${N0_COLOR}               - 0,1: when set to 1: prepare for acceptance of this
                     domain via live migration request ( used by 'bmigrate').
 ${N2_COLOR}lm_dport=${N0_COLOR}         - (optional) for live migration, port for migration.
                     data exchange. Can be '0' for auto port, by default.
 ${N2_COLOR}lm_rnodename=${N0_COLOR}     - (optional) for live migration, remote/source
                     CBSD nodename.
 ${N2_COLOR}quiet=${N0_COLOR}             - 0,1: be quiet, dont output verbose message.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd qstart

${H3_COLOR}See also${N0_COLOR}:

 cbsd qstop --help

"

CBSDMODULE="qemu"
EXTHELP="wf_xstop_qstart"

# by default - no live-migrated domain
lm="0"
lm_dport=
lm_rnodename=
odebug_engine=

cfg_only=		# create a configuration and run immediately

. ${subrdir}/nc.subr
. ${tools}		# for select_jail_by_list

readconf buildworld.conf
readconf jail-freebsd-default.conf

jname=

# check for cloud function when CBSDfile exist
Makefile="${CBSD_PWD}/CBSDfile"
if [ ! -r "${Makefile}" ]; then
	[ -z "${1}" ] && select_jail_by_list -s "List of offline VMs" -a "Off" -e qls -r ${sqlreplica}
fi

odebug_engine=		# overwrite debug_engine variable
debug_engine=		# reset debug_engine before init

delay=0
quiet=0
oquiet=0

. ${cbsdinit}
ojname="${jname}"

# live migration
if [ "${lm}" = "1" ]; then
	exec /usr/local/sbin/xl -t migrate-receive
	exit 0
fi

. ${system}
. ${subrdir}/universe.subr
. ${subrdir}/qemu.subr
. ${subrdir}/vnet.subr 		# get_vm_uplink_interface

if [ -r "${Makefile}" ]; then
	[ -z "${CBSDFILE_RECURSIVE}" -o ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}found CBSDfile: ${N2_COLOR}${Makefile}${N0_COLOR}" 1>&2
	. ${Makefile}
	all_qemu_list=$( ${GREP_CMD} -E '^qemu_[a-zA-Z0-9_@%:][-a-zA-Z0-9_@%:]*\(\)$' ${Makefile} | ${XARGS_CMD} | ${TR_CMD} -d "()" | ${SED_CMD} s#qemu_##g )
	jname="${all_qemu_list}"

	if [ -n "${ojname}" ]; then
		jname="${ojname}"
	fi

	[ -z "${jname}" ] && err 1 "${N1_COLOR}${CBSD_APP}: give me jname${N0_COLOR}"

	if [ -n "${CLOUD_URL}" -a -n "${CLOUD_KEY}" ]; then
		cbsd_api=1
	else
		cbsd_api=0
	fi
else
	cbsd_api=0
fi

# trim args from "$*"
ARGS=
for i in $*; do
	prefix6=$( substr --pos=0 --len=6 --str="${i}" )

	[ "${prefix6}" = "delay=" ] && shift && continue
	[ "${prefix6}" = "jname=" ] && shift && continue
	[ "${prefix6}" = "quiet=" ] && shift && continue
	[ "${prefix6}" = "inter=" ] && shift && continue

	if [ -n "${ARGS}" ]; then
		ARGS="${ARGS} ${i}"
	else
		ARGS="${i}"
	fi
done

[ -z "${jname}" -a -z "${ARGS}" ] && err 1 "${N1_COLOR}no qemu specified${N0_COLOR}"
[ -n "${debug_engine}" ] && odebug_engine="${debug_engine}"	# store overwrite debug_engine

. ${subrdir}/fetch.subr
. ${subrdir}/jcreate.subr       # for external_exec_master_script
. ${subrdir}/virtual.subr		# for init_systap

start_qemu()
{
	local _use_ovmf=1
	local QEMUCFG
	local xvm_ram _qemu_bin _log
	local _ebytes _dsk_ebytes _cloud_truncate
	local _cloud_source_is_zvol
	local _cloud_source_zvol _dsk_source_zvol _dsk_id
	local _label=
	local _zvol_pref=
	local _is_zvol=
	local _part=
	local _msdosfs=

	# profile
	readconf vm-${vm_os_type}-${vm_os_profile}.conf
	if [ -z "${vm_profile}" ]; then
		${ECHO} "${N1_COLOR}No such profile: ${N2_COLOR}vm-${vm_os_type}-${vm_os_profile}.conf${N0_COLOR}"
		sleep 2
	fi
	# re-read jail params and apply personal after profile
	. ${subrdir}/rcconf.subr

	# CBSD QUEUE
	if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
		[ -n "${cbsd_qemu_queue_name}" ] && ${cbsd_queue_backend} cbsd_qemu_queue_name=${cbsd_qemu_queue_name} id=${jname} cmd=qstart status=1 workdir="${workdir}"
	fi

	vm_boot=$( cbsdsqlro ${main_sqlite_local} SELECT vm_boot FROM settings 2>/dev/null )

	# todo: shared qemu/qemu virtual
	# Cloud-init init. Cloud init only for empty disk
	if check_for_empty_hdd path=${data}/dsk1.vhd; then
		if [ -n "${vm_iso_path}" ]; then
			local prefix=
			local prefix6=$( substr --pos=0 --len=6 --str="${vm_iso_path}" )
			if [ "${prefix6}" = "cloud-" ]; then
				local _orig_vm_iso_path="${vm_iso_path}"
				vm_boot="cd"
				init_iso
				if [ $? -eq 1 ]; then
					err 1 "${N1_COLOR}No such cloud source: ${N2_COLOR}${vm_iso_path}/${iso_img}${N0_COLOR}"
				fi
				vm_boot="hdd"
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}cloud init image initialization..${N0_COLOR}"
				local _myfile="${iso_img}"
				[ ! -r "${_myfile}" ] && err 1 "${N1_COLOR}Error: qstart: not readable: ${N2_COLOR}${_myfile}${N0_COLOR}"

				. ${subrdir}/zfs.subr

				if is_getzvol ${_myfile}; then
					_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_cloud_source_is_zvol=1
					_cloud_source_zvol="${is_zvol}"
				else
					_cloud_source_is_zvol=0
					_cloud_source_zvol=
					_ebytes=$( get_file_bytes ${_myfile} )
				fi

				# store original disk size to restore them after replace by cloud image
				if is_getzvol ${data}/dsk1.vhd; then
					_dsk_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_dsk_source_zvol="${is_zvol}"
					_cloud_truncate=0
				else
					_dsk_ebytes=$( get_file_bytes ${data}/dsk1.vhd )
					_cloud_truncate=1
					_dsk_source_zvol=
				fi

				# print some warning about not optimal when zfsfeat=1 but for some 
				# reason cloning not available?

				# if source image is not ZVOl, use dd method for cloning
				[ ${_cloud_source_is_zvol} -eq 0 ] && _cloud_truncate=1

				if [ ${_cloud_truncate} -eq 1 ]; then
					[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Clone cloud image into first/system vm disk (${W1_COLOR}dd${N1_COLOR} method)${N0_COLOR}"
					# to generic clonedata (add dd method)?
					# Linux does not support postfix in bs=, e.g. bs=4m
					case "${platform}" in
						Linux)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4000000 | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
						*)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4m | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
					esac
					echo
					# adjust original image size
					# not work when synlink to zvol
					[ ${quiet} -ne 1 ] && echo "${TRUNCATE_CMD} -s${_dsk_ebytes} ${data}/dsk1.vhd"
					${TRUNCATE_CMD} -s${_dsk_ebytes} ${data}/dsk1.vhd
				else
					[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Clone cloud image into first/system vm disk (zfs clone method)${N0_COLOR}"
					# to generic clonedata ?
						_cloud_snapshot_name="${_cloud_source_zvol}@boot-${jname}"
						echo "${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name}"
						${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name} > /dev/null 2>&1
						_ret=$?
						if [ ${_ret} -eq 1 ]; then
							# create cloud snapshot for $jname
							${ZFS_CMD} snapshot ${_cloud_source_zvol}@boot-${jname}
							# destory original zvol disk for vm
							${ZFS_CMD} destroy ${_dsk_source_zvol}
							#${ZFS_CMD} clone ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							echo "${ZFS_CMD} clone -o volsize=${_dsk_ebytes} ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}"
							${ZFS_CMD} clone -o volsize=${_dsk_ebytes} ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							case "${platform}" in
								Linux)
									# ?
									echo 'w' | fdisk zvol/${_dsk_source_zvol}
									true
									;;
								*)
									cbsdlogger NOTICE ${CBSD_APP}: zfs clone method cloud image: gpart commit zvol/${_dsk_source_zvol}
									${GPART_CMD} commit zvol/${_dsk_source_zvol}
							esac
							# restore original size
							# ${ZFS_CMD} set volsize=${_dsk_ebytes} ${_dsk_source_zvol}
							# bug here, need for atomic ops ^^ in clone action
							${ZFS_CMD} set cbsdsnap:jname=${jname} ${_cloud_source_zvol}@boot-${jname}
							${ZFS_CMD} set cbsdsnap:snapname=cloud ${_cloud_source_zvol}@boot-${jname}
						else
							err 1 "${N1_COLOR}snapshot already exist ${_cloud_snapshot_name}${N0_COLOR}"
						fi
				fi
				[ ${quiet} -ne 1 ] && echo "Eject cloud source: media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname}"
				media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname} quiet=${quiet}
			elif [ "${iso2img}" = "1" ]; then
				# Copy ISO as disk ?
				# Some profiles may use this, for example: Linux Tails
				local _orig_vm_iso_path="${vm_iso_path}"
				vm_boot="cd"
				init_iso
				if [ $? -eq 1 ]; then
					err 1 "${N1_COLOR}No such ISO source: ${N2_COLOR}${vm_iso_path}/${iso_img}${N0_COLOR}"
				fi
				vm_boot="hdd"
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ISO image initialization..${N0_COLOR}"
				local _myfile="${iso_img}"
				[ ! -r "${_myfile}" ] && err 1 "${N1_COLOR}Error: qstart: not readable: ${N2_COLOR}${_myfile}${N0_COLOR}"

				. ${subrdir}/zfs.subr

				if is_getzvol ${_myfile}; then
					_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_cloud_source_is_zvol=1
					_cloud_source_zvol="${is_zvol}"
				else
					_cloud_source_is_zvol=0
					_cloud_source_zvol=
					_ebytes=$( get_file_bytes ${_myfile} )
				fi

				# store original disk size to restore them after replace by cloud image
				if is_getzvol ${data}/dsk1.vhd; then
					_dsk_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_dsk_source_zvol="${is_zvol}"
					_cloud_truncate=0
				else
					_dsk_ebytes=$( get_file_bytes ${data}/dsk1.vhd )
					_cloud_truncate=1
					_dsk_source_zvol=
				fi

				# print some warning about not optimal when zfsfeat=1 but for some
				# reason cloning not available?

				# if source image is not ZVOl, use dd method for cloning
				# todo: really needed?
				[ ${_cloud_source_is_zvol} -eq 0 ] && _cloud_truncate=1

				if [ ${_cloud_truncate} -eq 1 ]; then
					[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ISO into first/system vm disk (${W1_COLOR}dd${N1_COLOR} method)${N0_COLOR}"
					# to generic clonedata (add dd method)?
					# Linux does not support postfix in bs=, e.g. bs=4m
					case "${platform}" in
						Linux)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4000000 | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
						*)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4m | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
					esac
					echo
					# adjust original image size.
					${TRUNCATE_CMD} -s${_dsk_ebytes} ${data}/dsk1.vhd
				else
					# to generic clonedata ?
					_cloud_snapshot_name="${_cloud_source_zvol}@boot-${jname}"
					echo "${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name}" 2>&1
					${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name} > /dev/null 2>&1
					_ret=$?
					if [ ${_ret} -eq 1 ]; then
						# create cloud snapshot for $jname
						${ZFS_CMD} snapshot ${_cloud_source_zvol}@boot-${jname}
						# destory original zvol disk for vm
						${ZFS_CMD} destroy ${_dsk_source_zvol}
						#${ZFS_CMD} clone ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
						${ZFS_CMD} clone -o volsize=${_dsk_ebytes} ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
						cbsdlogger NOTICE ${CBSD_APP}: zfs clone method cloud image: gpart commit zvol/${_dsk_source_zvol}
						${GPART_CMD} commit zvol/${_dsk_source_zvol}
						# restore original size
						# ${ZFS_CMD} set volsize=${_dsk_ebytes} ${_dsk_source_zvol}
						# bug here, need for atomic ops ^^ in clone action
						${ZFS_CMD} set cbsdsnap:jname=${jname} ${_cloud_source_zvol}@boot-${jname}
						${ZFS_CMD} set cbsdsnap:snapname=cloud ${_cloud_source_zvol}@boot-${jname}
					else
						err 1 "${N1_COLOR}snapshot already exist ${_cloud_snapshot_name}${N0_COLOR}"
					fi
				fi
#                                echo "Eject cloud source: media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname}" 2>&1
#                                media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname} quiet=${quiet} 2>&1
			else
				manage_boot_by_empty_hdd
			fi
		fi
	else
		manage_boot_by_empty_hdd
	fi

	if [ "${vm_boot}" = "cd" ]; then
		init_iso
		if [ $? -eq 1 ]; then
			printf "${N1_COLOR}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${N0_COLOR}"
				for i in $( ${JOT_CMD} 5 ); do
					[ ${quiet} -ne 1 ] && printf "."
					sleep 1
				done
				echo
		fi
	fi

	case "${vm_boot}" in
		"hdd")
			boot_arg='-boot c'
			;;
		"cd")
			boot_arg='-boot d'
			;;
	esac

	# for vnet we can make another action
	. ${subrdir}/vnet.subr

	#unset zero-value
	[ "${qemu_flags}" = "0" ] && unset qemu_flags
	[ "${vm_os_profile}" = "0" ] && unset vm_os_profile

	# reset global qemu_pci_id_busy_list (used by compile_ func via next_pci_id()
	# and remove buffer file with old $qemu_pci_id_busy_list ( used by add_qemu_pci_id_busy )
	qemu_pci_id_busy_list=

	qemu_pci_index=1
	[ -f ${jailsysdir}/${jname}/qemu_pciid ] && ${RM_CMD} -f ${jailsysdir}/${jname}/qemu_pciid

	# mark modified field to FALSE in pcibus table to find unmodified entries for pciid cleanup
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE pcibus SET modified=false

	# truncate pcibus_run table
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite DELETE FROM pcibus_run

	if ! compile_machine_args; then
		${ECHO} "${N1_COLOR}No such machine_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset machine_args
	fi
	if ! compile_cpu_args; then
		${ECHO} "${N1_COLOR}No such cpu_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset cpu_args
	fi
	if ! compile_kernel_args; then
		${ECHO} "${N1_COLOR}No such kernel_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset kernel_args
	fi
	if ! compile_bios_args; then
		${ECHO} "${N1_COLOR}No such bios_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset bios_args
	fi
	if ! compile_vga_args; then
		${ECHO} "${N1_COLOR}No such vga_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset vga_args
	fi
	if ! compile_dsk_args; then
		${ECHO} "${N1_COLOR}No such disk for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset dsk_args
	fi

	if ! compile_cd_args; then
		unset cd_args
	fi

	# init nic_args
	if ! compile_nic_args ; then
		${ECHO} "${N1_COLOR}No such nic for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset nic_args
	fi

	# init console_args
	#if ! compile_console_args; then
	#	${ECHO} "${N1_COLOR}No such console for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
	#	unset console_args
	#fi

	if [ "${nographic}" != "1" ]; then
		if [ "${spice_default}" != "1" ]; then
			# init vnc_args
			if ! compile_vnc_args ; then
				unset vnc_args
			fi
		else
			# init vnc_args
			if ! compile_spice_args ; then
				unset vnc_spice
			fi
		fi
	else
		unset vnc_args vnc_spice
		vga_args="-nographic"
	fi
	xvm_ram=$(( vm_ram / 1024 / 1024  ))

	# Poehali!

	export_qemu_data_for_external_hook
	external_exec_master_script "master_prestart.d"

	vm_logfile=$( ${MKTEMP_CMD} )

	QEMUCFG="${jailsysdir}/${jname}/qemu.cfg"
	${TRUNCATE_CMD} -s0 ${QEMUCFG}

	# restore overwrite debug_engine
	[ -n "${odebug_engine}" ] && debug_engine="${odebug_engine}"

	# todo: loop for multiple devices
	eval mybridge="\$nic0"

	case ${spice_default} in
		0)
			# init args
			;;
		1)
			# init args
			;;
	esac

#	if [ -n "${soundhw}" -a "${soundhw}" != "none" ]; then
#		${CAT_CMD} >> ${QEMUCFG} <<EOF
#soundhw="${soundhw}"
#EOF
#	fi

	# UEFI
	ovmf_firmware=

	case "${vm_arch}" in
		aarch64)
			local _valid_ovmf_files="FVP_AARCH64_EFI.fd QEMU_EFI.fd"
			local _valid_ovmf_path_dir="/usr/local/share/edk2-fvp /usr/share/edk2-fvp /usr/share/qemu-efi-aarch64 /usr/local/share/qemu-efi-aarch64 ${workdir}/share/qemu-efi-aarch64"
			_use_ovmf=0
			;;
		riscv64)
			_use_ovmf=0
			;;
		*)
			local _valid_ovmf_files="OVMF_CODE_4M.fd OVMF.fd OVMF_CODE.fd QEMU_UEFI-x86_64.fd QEMU_UEFI_CODE-x86_64.fd"
			local _valid_ovmf_path_dir="/usr/share/OVMF /usr/share/ovmf /usr/share/edk2/x64 /usr/share/edk2-qemu/x64 /usr/local/share/edk2-qemu ${workdir}/share/qemu-efi-x64"
			_use_ovmf=1
			;;
	esac

	if [ ${_use_ovmf} -eq 1 ]; then
		for i in ${_valid_ovmf_files}; do
			[ -n "${ovmf_firmware}" ] && break
			for j in ${_valid_ovmf_path_dir}; do
				[ -n "${ovmf_firmware}" ] && break
				[ -r "${j}/${i}" ] && ovmf_firmware="${j}/${i}"
			done
		done

		if [ -r "${ovmf_firmware}" ]; then
			# UEFI BOOT
			uefi_args="-drive if=pflash,format=raw,unit=0,readonly=on,file=${ovmf_firmware}"
			uefi_args="${uefi_args} -drive if=pflash,format=raw,unit=1,file=${jailsysdir}/${jname}/BHYVE_UEFI_VARS.fd"
		else
			uefi_args=
		fi
	else
		uefi_args=
	fi

	if [ ${quiet} -ne 1 ]; then
		case "${vm_arch}" in
			aarch64|riscv64)
				true
				;;
			*)
				[ -z "${uefi_args}" ] && ${ECHO} "${N1_COLOR}UEFI firmware not found ('edk2-qemu-x64' for x86, 'edk2-fvp' for aarch64): UEFI boot disabled${N0_COLOR}"
				;;
		esac
	fi
	# todo: shared qemu/qemu virtual
	# cloud-init enabled?
	# this section should be AFTER master_prestart
	# cloud_init and CLOUD_FILES was initialized earlier
	if [ ${cloud_init} -eq 1 ]; then
		case "${ci_template}" in
			fire*)
				${CAT_CMD} ${jailsysdir}/${jname}/cloud-init/* > ${data}/dsk2.vhd
				${TRUNCATE_CMD} -s1m ${data}/dsk2.vhd
				;;
			homeass)
				. ${subrdir}/zfs.subr
				if is_getzvol ${data}/dsk1.vhd; then
					zvol_dsk0_dev=$( ${READLINK_CMD} ${data}/dsk1.vhd )
					zvol_dsk0=$( echo ${zvol_dsk0_dev} | ${SED_CMD} s#/dev/zvol/## )
					${ZFS_CMD} set volmode=full ${zvol_dsk0}
					case "${platform}" in
						Linux)
							# needed?
							#echo 'w' | fdisk /dev/zvol/${zvol_dsk0}
							true
							;;
						*)
							${GPART_CMD} recover ${zvol_dsk0_dev}
							;;
					esac
				else
					case "${platform}" in
						Linux)
							zvol_dsk0_dev=$( ${LOSETUP_CMD} --find --show ${data}/dsk1.vhd )
							_ret=$?
							if [ ${_ret} -ne 0 ]; then
								err 1 "${N1_COLOR}${CBSD_APP}: homeass cloud-init: error: ${N2_COLOR}${LOSETUP_CMD} --find --show ${data}/dsk1.vhd${N0_COLOR}"
							fi
							partprobe ${zvol_dsk0_dev}
							;;
						*)
							zvol_dsk0_dev=$( ${MDCONFIG_CMD} -a -t vnode -f ${data}/dsk1.vhd )
							_ret=$?
							if [ ${_ret} -ne 0 ]; then
								err 1 "${N1_COLOR}${CBSD_APP}: homeass cloud-init: error: ${N2_COLOR}${MDCONFIG_CMD} -a -t vnode -f ${data}/dsk1.vhd${N0_COLOR}"
							fi
							${GPART_CMD} recover ${zvol_dsk0_dev}
							zvol_dsk0_dev="/dev/${zvol_dsk0_dev}"
					esac
				fi

				mount_dir=$( ${MKTEMP_CMD} -d )
				case "${platform}" in
					Linux)
						case ${zvol_dsk0_dev} in
							/dev/loop*)
								_part="p1"
								;;
							*)
								_part="-part1"
								;;
						esac
						_msdosfs="vfat"
						;;
					*)
						_part="p1"
						_msdosfs="msdosfs"
						;;
				esac
				${MOUNT_CMD} -t ${_msdosfs} -o rw ${zvol_dsk0_dev}${_part} ${mount_dir}
				_ret=$?
				if [ ${_ret} -ne 0 ]; then
					err 1 "${N1_COLOR}${CBSD_APP}: homeass cloud-init: error: ${N2_COLOR}${MOUNT_CMD} -t msdosfs -o rw ${zvol_dsk0_dev}p1 ${mount_dir}${N0_COLOR}"
				fi
				${MKDIR_CMD} -p ${mount_dir}/CONFIG/network
				${CP_CMD} -a ${jailsysdir}/${jname}/cloud-init/* ${mount_dir}/CONFIG/network/
				${LS_CMD} -l ${mount_dir}/CONFIG/network/
				${CAT_CMD} -n ${mount_dir}/CONFIG/network/*
				${CP_CMD} -HL ${workdir}/.ssh/id_rsa.pub ${mount_dir}/CONFIG/authorized_keys

				${UMOUNT_CMD} ${mount_dir}
				${RMDIR_CMD} ${mount_dir}

				case "${zvol_dsk0_dev}" in
					md*|/dev/loop*)
						case "${platform}" in
							Linux)
								${LOSETUP_CMD} --detach ${zvol_dsk0_dev}
								;;
							*)
								${MDCONFIG_CMD} -d -u ${zvol_dsk0_dev}
								;;
						esac
						;;
				esac
				;;
			*)
				${RM_CMD} -f ${jailsysdir}/${jname}/seed.iso
				if [ "${vm_os_type}" = "windows" ]; then
					GENISOIMAGE_CMD=$( which /usr/local/bin/genisoimage )
					[ -z "${GENISOIMAGE_CMD}" ] && err 1 "${N1_COLOR}no such genisoimage tool. Windows cloud required it. Please install it first: ${N2_COLOR}pkg install -y sysutils/genisoimage${N0_COLOR}"
					${GENISOIMAGE_CMD} -joliet-long -R -V config-2 -o ${jailsysdir}/${jname}/seed.iso ${jailsysdir}/${jname}/cloud-init
				else
					case "${platform}" in
						Linux)
							GENISOIMAGE_CMD=$( which xorriso )
							_log=$( ${MKTEMP_CMD} )

							case "${ci_template}" in
								*)
									_label="CIDATA"
									;;
							esac

							${GENISOIMAGE_CMD} \
								-as mkisofs \
								-iso-level 2 \
								-volid ${_label} \
								-output "${jailsysdir}/${jname}/seed.iso" \
								-joliet \
								-rational-rock \
								"${jailsysdir}/${jname}/cloud-init" > ${_log} 2>&1
							ret=$?

							if [ ${ret} -ne 0 ]; then
								${CAT_CMD} ${_log}
								err 1 "xorriso error"
							fi
							${RM_CMD} -f ${_log}
							true
							;;
						*)
							GENISOIMAGE_CMD=$( which makefs )
							case "${ci_template}" in
								*)
									_label="cidata"
									;;
							esac
							${GENISOIMAGE_CMD} -t cd9660 -o label="${_label}" -o isolevel=2 -o rockridge -o publisher="CBSD" ${jailsysdir}/${jname}/seed.iso ${jailsysdir}/${jname}/cloud-init
							# see /usr/src/usr.sbin/makefs/cd9660/cd9660_strings.c + /usr/src/usr.sbin/makefs/cd9660.c (cd9660_valid_a_char)
							# why upper here, whereis spec?
							${SED_CMD} -i${SED_DELIMER}'' s:CIDATA:cidata: ${jailsysdir}/${jname}/seed.iso
							;;
					esac
				fi
			;;
		esac
	fi

	case "${arch}" in
		riscv64)
			_qemu_bin=$( which qemu-system-${arch} 2>/dev/null )
			[ -z "${_qemu_bin}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such executable: ${N2_COLOR}qemu-system-${arch}${N0_COLOR}"
			;;
		aarch64)
			_qemu_bin=$( which qemu-system-${arch} 2>/dev/null )
			[ -z "${_qemu_bin}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such executable: ${N2_COLOR}qemu-system-${arch}${N0_COLOR}"
			;;
		*)
			_qemu_bin="${QEMU_SYSTEM_X86_64_CMD}"
			[ -z "${_qemu_bin}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such executable: ${N2_COLOR}qemu-system-${arch}${N0_COLOR}"
			;;
	esac

# compile_rnd
#-object rng-random,id=rng0,filename=/dev/urandom \
#-device virtio-rng-pci,rng=rng0 \

# compile_usb
#-usb -device qemu-xhci,id=usbbus \

display_args="-display none"
#display_args="-display curses"
#display_args="-display sdl"

	# qemu restore/checkpoint
	if [ -n "${checkpoint}" ]; then
		CHECKPOINT_DIR="${jailsysdir}/${jname}/checkpoints"
		CHECKPOINT="${CHECKPOINT_DIR}/${checkpoint}.ckp"
		if [ -r ${CHECKPOINT} ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Checkpoint found, starting from: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
			qemu_cmd="${XL_CMD} -vvv restore ${CHECKPOINT}"
		else
			err 1 "${N1_COLOR}Checkpoint not found: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
		fi
	else
		# -machine type=q35 \
		#-cpu host -M pc-i440fx-hirsute
		#-cpu host \
		qemu_cmd="${_qemu_bin} \
${machine_args} \
${cpu_args} \
-smp cpus=${vm_cpus} -m ${xvm_ram} \
${uefi_args} \
${kernel_args} \
${bios_args} \
${cd_args} \
${boot_arg} \
${dsk_args} \
${nic_args} \
${vnc_args} \
${display_curses} \
${vga_args} \
-name ${jname} \
-pidfile ${jailsysdir}/${jname}/qemu.pid \
-monitor unix:${jailsysdir}/${jname}/qemu-monitor.sock,server,nowait"
	fi

	[ ${quiet} -ne 1 ] && echo "[debug]: ${qemu_cmd}"
	[ -n "${cfg_only}" ] && exit 0

	# API compat: ~cbsd/jails-system/info*.*
	qemu_cmd="${qemu_cmd}; /usr/local/cbsd/misc/daemonize /usr/local/bin/cbsd /usr/local/cbsd/tools/save-jail-info jname=${jname}"

	case "${debug_engine}" in
		gdb)
			if [ -x /usr/local/bin/gdb ]; then
				gdb_cmd="/usr/local/bin/gdb"
			elif [ -x /usr/libexec/gdb ]; then
				gdb_cmd="/usr/libexec/gdb"
			elif [ -x /usr/bin/gdb ]; then
				gdb_cmd="/usr/bin/gdb"
			fi
			# break while loop
			echo
			echo "Warning"
			echo "Run xl throuch GDB. Please execute 'run' to launch QEMU instance"
			echo
			echo "${gdb_cmd} -batch --args ${qemu_cmd}"
			${gdb_cmd} -ex run --args ${qemu_cmd}
			;;
		lldb)
			echo
			echo "Warning"
			echo "Run xl throuch LLDB. Please execute 'run' to launch QEMU instance"
			echo
			echo "/usr/bin/lldb -- ${qemu_cmd}"
			/usr/bin/lldb -- ${qemu_cmd}
			qemu_exit=$?
			${RM_CMD} -f ${tmpdir}/cmds.$$
			;;
		*)
			tmuxcmd=$( which tmux )
			[ -z "${tmuxcmd}" ] && err 1 "${N1_COLOR}bstart: no such tmux cmd${N0_COLOR}"
			#${tmuxcmd} -2 -u new -d -s "cbsd-${jname}" "/bin/sh ${jailsysdir}/${jname}/run.sh"
			${tmuxcmd} -2 -u new -d -s "cbsd-${jname}" "${qemu_cmd}"
			;;
	esac

	[ -n "${dsk_bootable}" -o ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Boot device: ${N2_COLOR}${dsk_bootable}${N0_COLOR}"
	[ ${quiet} -ne 1 ] && printf "${N1_COLOR}Waiting for PID"
	vm_pid=0
	for i in $( ${SEQ_CMD} 10 ); do
		if [ -r ${jailsysdir}/${jname}/qemu.pid ]; then
			_test_pid=$( ${CAT_CMD} ${jailsysdir}/${jname}/qemu.pid 2>/dev/null )
			if [ -n "${_test_pid}" ]; then
				${PS_CMD} -p ${_test_pid} > /dev/null 2>&1
				_ret=$?
				if [ ${_ret} -eq 0 ]; then
					vm_pid="${_test_pid}"
					break
				fi
			fi
		fi
		sleep 1
		[ ${quiet} -ne 1 ] && printf "."
	done

	[ -z "${vm_pid}" ] && vm_pid="0"

	[ ${quiet} -ne 1 ] && echo

	external_exec_master_script "master_poststart.d"

	# CBSD QUEUE
	if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
		[ -n "${cbsd_qemu_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_qemu_queue_name} id=${jname} cmd=bstart status=2 data_status=1 workdir="${workdir}"
	fi

	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}PID: ${N2_COLOR}${vm_pid}${N0_COLOR}"
	cbsdsqlrw local "UPDATE jails SET jid='${vm_pid}' WHERE jname='${jname}'"

	# update state_time
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET state_time="(strftime('%s','now'))"

	# update state_time, local SQLite for back compatible
	cbsdsqlrw local UPDATE jails SET state_time="(strftime('%s','now'))" WHERE jname='${jname}'

	expose mode=apply

	# first-run hack for MYbQT via SSH method.
	if [ -r ${jailsysdir}/${jname}/mybqt ]; then
		${RM_CMD} ${jailsysdir}/${jname}/mybqt
		nodeip-expose mode=create jname=${jname}
	fi

	return 0
}


# MAIN for multiple jails
TRAP=""
emulator="qemu"		# for jname_is_multiple
jname_is_multiple
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

if [ $# -gt 1 -a -z "${jname}" -o -n "${jail_list}" ]; then
	# multiple astart always non interactive
	export inter=0
	# recursive
	if [ -n "${jail_list}" ]; then
		JLIST="${jail_list}"
	else
		JLIST=$*
	fi

	for jname in ${JLIST}; do
		TRAP="${TRAP} ${RM_CMD} -f ${ftmpdir}/qstart.${jname}.$$;"
		trap "${TRAP}" HUP INT ABRT BUS TERM EXIT
		env NOINTER=1 /usr/local/cbsd/misc/daemonize -E NOINTER=1 -p ${ftmpdir}/qstart.${jname}.$$ /usr/local/bin/cbsd qstart jname=${jname} delay=1
		#lets save .pid file
		sleep 1
		[ -f "${ftmpdir}/qstart.${jname}.$$" ] && cbsd_pwait --pid=$( ${CAT_CMD} ${ftmpdir}/qstart.${jname}.$$ ) --timeout=${parallel}
		trap "" HUP INT ABRT BUS TERM EXIT
		# Artificial delay to create a sequence (for order compliance)
		# todo: determine VM complete starting
		sleep 12
	done

	wait_for_fpid -a start -t ${parallel}

	err 0 "${N1_COLOR}Multiple qstart: ${N2_COLOR}done${N0_COLOR}"
fi

# MAIN
. ${distsharedir}/qemu.conf		# only for for MYCOL variables: used in exports below
. ${subrdir}/time.subr
[ -z "${jname}" ] && jname=${1}

init_qemu

[ "${create_cbsdsystem_tap}" = "1" ] && init_systap

# mainly to smooth mass start in astart.
if [ ${delay} -ne 0 ]; then
	[ -z "${boot_delay}" ] && boot_delay=1
		cbsdlogger NOTICE ${CBSD_APP}: delayed boot for ${jname}: ${boot_delay} sec.
		sleep ${boot_delay}
fi

# start qemu
st_time=$( ${DATE_CMD} +%s )

# cloud-init enabled?
# we need this section (besides section cloud-init section in start_qemu() to allocate pci bus id via touching seed.iso
# since some helpers may work with runtime (e.g pcibus + cloudinit) config
if [ -d ${jailsysdir}/${jname}/cloud-init ]; then
	CLOUD_FILES=$( ${FIND_CMD} ${jailsysdir}/${jname}/cloud-init/ -mindepth 1 -type f | ${XARGS_CMD} )
	# gen seed only if files exist
	if [ -n "${CLOUD_FILES}" ]; then
		cloud_init=1
		[ ${quiet} -ne 1 ] && ${ECHO} "${H5_COLOR}cloud-init: ${H3_COLOR}enabled${N0_COLOR}"
		${TOUCH_CMD} ${jailsysdir}/${jname}/seed.iso
	else
		cloud_init=0
		unset CLOUD_FILES
	fi
else
	cloud_init=0
fi


default_profile="qemu-default-default.conf"
readconf vnc.conf
readconf spice.conf
readconf qstart.conf

readconf ${default_profile}

. ${subrdir}/rcconf.subr
[ $? -eq 1 ] && err 1 "${N1_COLOR}No such domain: ${N2_COLOR}${jname}${N0_COLOR}"
[ ${status} -eq 2 ] && err 1 "${N1_COLOR}Domain in slave mode. Please ${N2_COLOR}cbsd jswmode mode=master${N1_COLOR} first${N0_COLOR}"

[ ${jid} -ne 0 ] && err 1 "${N1_COLOR}Jail ${jname} already running, jid: ${N2_COLOR}${jid}${N0_COLOR}"
[ "${emulator}" != "qemu" ] && err 1 "${N1_COLOR}Not qemu mode${N0_COLOR}"

# cleanup old artifacts before start new session
qcleanup jname="${jname}"

[ -z "${vm_ram}" -o -z "${vm_cpus}" -o -z "${vm_os_type}" ] && err 1 "${N1_COLOR}Parameter is mandatory: ${N2_COLOR}vm_ram, vm_cpus, vm_os_type${N0_COLOR}"
[ -z "${iso_auto_fetch}" ] && iso_auto_fetch=0
[ -z "${debug}" ] && debug=0

[ -n "${ointer}" ] && inter="${ointer}"
[ -n "${inter}" ] && export NOINTER=1

if [ ${vm_cpus} -gt ${ncpu} -a ${vm_cpus} -lt ${vm_cpus_max} ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Warning! Current node cpu: ${N2_COLOR}${ncpu}${N1_COLOR}, guest cpu: ${N2_COLOR}${vm_cpus}. ${N1_COLOR}Overcommitting vCPUs can hurt perfomance.${N0_COLOR}"
elif [ ${vm_cpus} -lt 1 -o ${vm_cpus} -gt ${vm_cpus_max} ]; then
	err 1 "${N1_COLOR}Valid number of guest CPUs within 1 - ${vm_cpus_max} range. Current vm_cpus: ${N2_COLOR}${vm_cpus}${N0_COLOR}"
fi

main_sqlite_local="${jailsysdir}/${jname}/local.sqlite"

# hardcoded first disk path from SQL. Todo: mark bootable disk(s)
MDFILE=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_path FROM qemudsk WHERE jname=\"${jname}\" AND dsk_type=\"vhd\" LIMIT 1 2>/dev/null )

if [ -z "${MDFILE}" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Warning: no any storage device found for this VM${N0_COLOR}"
else
	if [ ! -f "${data}/${MDFILE}" -a ! -h "${data}/${MDFILE}" ]; then
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}No such ${data}/${MDFILE} but mdsize flags is not null.${N0_COLOR}"

		# if zfsfeat=1, try scan for zvol
		[ "${zfsfeat}" != "1" ] && break

		readconf zfs.conf
		. ${subrdir}/zfs.subr
		DATA=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} 2>/dev/null )

		[ -z "${DATA}" ] && break

		for lunname in $( ${SEQ_CMD} 0 10 ); do
			if [ -r /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ]; then
				${LN_CMD} -sf /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ${data}/dsk${lunname}.vhd
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Found zvol and create symlink: ${data}/dsk${lunname}.vhd -> ${DATA}/bcbsd-${jname}-dsk${lunname}.vhd"
			fi
		done
	fi
fi

# export variables for external hooks
export jname=${jname}

for _i in ${JARG} ${MYCOL}; do
	T=
	eval T="\$$_i"
	export ${_i}="${T}"
done

# test for incorrect state
if [ ${status} -eq 3 ]; then
	cbsdsqlrw local UPDATE jails SET maintenance=\"${comment}\" WHERE jname=\"${jname}\"
	comment="cbsdsqlro local SELECT maintenance FROM jails WHERE jname=\"${jname}\""
	if [ "${comment}" = "Stopping_VM" ]; then
		jswmode jname=${jname} mode=master comment='0'
	else
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Xen in maintenance: ${N2_COLOR}${comment}${N0_COLOR}"
		err 1 "${N1_COLOR}Please finish maintenance and switch mode via: ${N2_COLOR}jswmode jname=${jname} mode=master comment='0'${N0_COLOR}"
	fi
fi

start_qemu

# API compat: ~cbsd/jails-system/info*.*
/usr/local/cbsd/misc/daemonize /usr/local/cbsd/tools/save-jail-info jname=${jname}

end_time=$( ${DATE_CMD} +%s )
diff_time=$(( end_time - st_time ))
diff_time=$( displaytime ${diff_time} )
cbsdlogger NOTICE ${CBSD_APP}: QEMU domain ${jname} started in ${diff_time}

exit 0
