#!/usr/local/bin/cbsd
#v12.1.16
MYARG=""
MYOPTARG="bhyverun_wrapper cfg_only checkpoint debug debug_engine delay extra_profile_dir inter jname lm lm_dport lm_rnodename progress_state_file quiet"
MYDESC="Start bhyve domain"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

Start the bhyve domain. You can start the offline machine, or continue the work
of the frozen machine from the checkpoint.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}bhyverun_wrapper=${N0_COLOR} - <file_path>, alternative/custom bhyve run script.
 ${N2_COLOR}cfg_only=${N0_COLOR}         - <file_path>: when set, find/create tap/vnc and
                     generate bhyve.conf into <file_path> but without
                     run, e.g for debugging.
 ${N2_COLOR}checkpoint=${N0_COLOR}       - start from specified checkpoint name.
 ${N2_COLOR}debug=${N0_COLOR}            - more debug messages.
 ${N2_COLOR}debug_engine=${N0_COLOR}     - overwrite debug_engine settings: use gdb or lldb
                     as debugger when launch bhyve ( mostly for inherits debug
                     with live migration ).
 ${N2_COLOR}delay=${N0_COLOR}            - <sec>, delay N secbefore start, mainly to smooth 
                     the astart, default is: '0', no delay.
 ${N2_COLOR}extra_profile_dir${N0_COLOR}   - Extra directory to search VMs profiles.
 ${N2_COLOR}inter=${N0_COLOR}            - set 1 to prevent any questions and to accept answers by default.
 ${N2_COLOR}jname=${N0_COLOR}            - Target VM. If jail='*' or jail='vm*' then start  all
                     bhyve or VM whose names begin with 'vm', e.g. 'vm1', 'vm2'...
 ${N2_COLOR}lm=${N0_COLOR}               - 0,1: when set to 1: prepare for acceptance of this
                     domain via live migration request ( used by 'bmigrate').
 ${N2_COLOR}lm_dport=${N0_COLOR}         - (optional) for live migration, port for migration.
                     data exchange. Can be '0' for auto port, by default.
 ${N2_COLOR}lm_rnodename=${N0_COLOR}     - (optional) for live migration, remote/source
                     CBSD nodename.
 ${N2_COLOR}quiet=${N0_COLOR}            - 0,1: be quiet, dont output verbose message.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd bstart
 # cbsd bstart jname=myvm1 checkpoint=point1

${H3_COLOR}See also${N0_COLOR}:

 cbsd bstop --help
 cbsd bcheckpoint --help

"

CBSDMODULE="bhyve"
EXTHELP="wf_bstop_bstart"

# by default - no live-migrated domain
lm="0"
lm_dport=
lm_rnodename=
odebug_engine=

cfg_only=		# create a configuration and run immediately

. ${subrdir}/nc.subr
. ${tools}		# for select_jail_by_list
. ${subrdir}/ipfw.subr	# fwcounter

readconf buildworld.conf
readconf jail-freebsd-default.conf

oextra_profile_dir=
extra_profile_dir=
readconf bhyve-default-default.conf

jname=

# check for cloud function when CBSDfile exist
Makefile="${CBSD_PWD}/CBSDfile"
if [ ! -r "${Makefile}" ]; then
	[ -z "${1}" ] && select_jail_by_list -s "List of offline VMs" -a "Off" -e bls -r ${sqlreplica}
fi

odebug_engine=		# overwrite debug_engine variable
debug_engine=		# reset debug_engine before init
obhyverun_wrapper=	# reset bhyverun_wrapper
bhyverun_wrapper=	# reset bhyverun_wrapper

delay=0
odelay=
progress_state_file=
quiet=0
oquiet=0
. ${cbsdinit}
[ -n "${extra_profile_dir}" ] && oextra_profile_dir="${extra_profile_dir}"
ojname="${jname}"
[ -n "${delay}" ] && odelay="${delay}"

. ${system}
. ${subrdir}/universe.subr
. ${subrdir}/bhyve.subr
. ${subrdir}/vnet.subr		# for get_vm_uplink_interface

# adjust bhyve_list by CBSDfile
if [ -r "${Makefile}" ]; then
	[ -z "${CBSDFILE_RECURSIVE}" ] && ${ECHO} "${N1_COLOR}found CBSDfile: ${N2_COLOR}${Makefile}${N0_COLOR}" 1>&2
	. ${Makefile}
	all_bhyve_list=$( ${GREP_CMD} -E '^bhyve_[a-zA-Z0-9_@%:][-a-zA-Z0-9_@%:]*\(\)$' ${Makefile} | ${XARGS_CMD} | ${TR_CMD} -d "()" | ${SED_CMD} s#bhyve_##g )
	jname="${all_bhyve_list}"

	# cbsd jstart <env1> <env2> route
	jname_in_args=0

	if [ -n "${ojname}" ]; then
		bhyve_list="${ojname}"
	else
		# cbsd bstart <env1> <env2> route
		for i in $*; do
			strpos --str="${i}" --search="="
			if [ $? -eq 0 ]; then
				jname_in_args=1
				# not param=value - jail?
				jname_in_cbsdfile=0
				for j in ${all_bhyve_list}; do
					[ "${i}" != "${j}" ] && continue
					jname_in_cbsdfile=1
				done
				if [ ${jname_in_cbsdfile} -eq 1 ]; then
					if [ -z "${ojname}" ]; then
						ojname="${i}"
					else
						ojname="${ojname} ${i}"
					fi
				else
					${ECHO} "${N1_COLOR}${CBSD_APP}: env absent in CBSDfile: ${N2_COLOR}${i}${N0_COLOR}" 1>&2
				fi
			fi
		done
		if [ ${jname_in_args} -eq 1 ]; then
			bhyve_list="${ojname}"
		else
			bhyve_list="${all_bhyve_list}"
		fi
	fi

	unset ojname jname

	[ -z "${bhyve_list}" ] && err 1 "${N1_COLOR}${CBSD_APP}: give me jname${N0_COLOR}"

	# multiple?
	strpos --str="${jname}" --search=" "
	if [ $? -ne 0 ]; then
		# alone
		jname="${bhyve_list}"
		unset bhyve_list
	fi

	if [ -n "${CLOUD_URL}" -a -n "${CLOUD_KEY}" ]; then
		cbsd_api=1
	else
		cbsd_api=0
	fi
else
	cbsd_api=0
	# trim args from "$*"

	if [ -n "${ojname}" ]; then
		bhyve_list="${ojname}"
	else
		for i in $*; do
			strpos --str="${i}" --search="="
			[ $? -ne 0 ] && continue

			if [ -z "${bhyve_list}" ]; then
				bhyve_list="${i}"
			else
				bhyve_list="${bhyve_list} ${i}"
			fi
		done
	fi

	# multiple?
	strpos --str="${bhyve_list}" --search=" "
	if [ $? -eq 0 ]; then
		# alone
		jname="${bhyve_list}"
		unset bhyve_list
	fi
fi

[ -z "${jname}" -a -z "${bhyve_list}" ] && err 1 "${N1_COLOR}No vm specified${N0_COLOR}"
[ -n "${debug_engine}" ] && odebug_engine="${debug_engine}"		# store overwrite debug_engine
[ -n "${bhyverun_wrapper}" ] && obhyverun_wrapper="${bhyverun_wrapper}"	# store overwrite bhyverun_wrapper

. ${subrdir}/fetch.subr
. ${subrdir}/jcreate.subr	# for external_exec_master_script
. ${subrdir}/virtual.subr	# for init_systap
. ${subrdir}/time.subr

create_from_jail()
{
	local passt_args=

	[ -n "${pci_passthru_args}" ] && passt_args="-S"

	# from jail2iso mode
	case "${vm_efi}" in
		uefi_csm|uefi)
			bhyveload_cmd=""
		;;
		*)
			bhyveload_cmd="${LOCKF_CMD} -s -t0 ${tmpdir}/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -m ${vm_ram} -d ${data}/${MDFILE} ${jname}"
		;;
	esac
}

start_bhyve()
{
	local bhyveload_cmd _err
	local passt_args=
	local _ebytes _dsk_ebytes _cloud_truncate
	local _cloud_source_is_zvol
	local _cloud_source_zvol _dsk_source_zvol _dsk_id
	local _label=
	local _zvol_pref=
	local _is_zvol=

	local is_ppt=$( cbsdsqlro ${main_sqlite_local} "SELECT ppt FROM bhyveppt WHERE jname='${jname}'" 2>/dev/null | ${AWK_CMD} '{printf $1}' )
	[ -n "${is_ppt}" ] && passt_args="-S"

	# flags which means that the CD as boot device will use only once
	cd_boot_once=0

	# profile
	readconf vm-${vm_os_type}-${vm_os_profile}.conf

	[ -n "${oextra_profile_dir}" ] && extra_profile_dir="${oextra_profile_dir}"

	if [ -n "${extra_profile_dir}" ]; then
		if [ ! -d "${extra_profile_dir}" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${W1_COLOR}${CBSD_APP} warning: ${N1_COLOR}no such extra_profile_dir: ${N2_COLOR}${extra_profile_dir}${N0_COLOR}" >&2
		else
			[ -r ${extra_profile_dir}/vm-${vm_os_type}-${vm_os_profile}.conf ] && . ${extra_profile_dir}/vm-${vm_os_type}-${vm_os_profile}.conf
		fi
	fi

	if [ -z "${vm_profile}" ]; then
		${ECHO} "${N1_COLOR}No such profile: ${N2_COLOR}vm-${vm_os_type}-${vm_os_profile}.conf${N0_COLOR}"
		sleep 2
	fi
	# re-read jail params and apply personal after profile
	if [ "${lm}" = "0" ]; then
		. ${subrdir}/rcconf.subr
	else
		. ${subrdir}/rrcconf.subr
	fi

	# CBSD QUEUE
	if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
		[ -n "${cbsd_bhyve_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_bhyve_queue_name} id=${jname} cmd=bstart status=1 workdir="${workdir}"
	fi

	vm_boot=$( cbsdsqlro ${main_sqlite_local} SELECT vm_boot FROM settings 2>/dev/null | ${AWK_CMD} '{printf $1}' )
	bhyve_cmd=$( cbsdsqlro ${main_sqlite_local} SELECT bhyve_cmd FROM settings 2>/dev/null | ${AWK_CMD} '{printf $1}' )

	[ ! -x ${bhyve_cmd} ] && err 1 "${N1_COLOR}bstart: bhyve cmd not executable: ${N2_COLOR}${bhyve_cmd}${N1_COLOR}. Please set proper bhyve_cmd${N0_COLOR}"

	# live migration todo:
	# a) no cdrom (error?)
	# b) shared ISO dir too?

	# todo: shared bhyve/xen virtual
	# Cloud-init init. Cloud init only for empty disk
	if check_for_empty_hdd path=${data}/dsk1.vhd; then
		if [ -n "${vm_iso_path}" ]; then
			local prefix=
			local prefix6=$( substr --pos=0 --len=6 --str="${vm_iso_path}" )

			if [ "${prefix6}" = "cloud-" ]; then
				local _orig_vm_iso_path="${vm_iso_path}"
				vm_boot="cd"
				init_iso
				if [ $? -eq 1 ]; then
					err 1 "${N1_COLOR}No such cloud source: ${N2_COLOR}${vm_iso_path}/${iso_img}${N0_COLOR}"
				fi
				vm_boot="hdd"
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}cloud init image initialization..${N0_COLOR}"
				local _myfile="${iso_img}"
				[ ! -r "${_myfile}" ] && err 1 "${N1_COLOR}Error: bstart: not readable: ${N2_COLOR}${_myfile}${N0_COLOR}"

				. ${subrdir}/zfs.subr

				if is_getzvol ${_myfile}; then
					_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_cloud_source_is_zvol=1
					_cloud_source_zvol="${is_zvol}"
				else
					_cloud_source_is_zvol=0
					_cloud_source_zvol=
					_ebytes=$( get_file_bytes ${_myfile} )
				fi

				# store original disk size to restore them after replace by cloud image
				if is_getzvol ${data}/dsk1.vhd; then
					_dsk_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_dsk_source_zvol="${is_zvol}"
					_cloud_truncate=0
				else
					_dsk_ebytes=$( get_file_bytes ${data}/dsk1.vhd )
					_cloud_truncate=1
					_dsk_source_zvol=
				fi

				# print some warning about not optimal when zfsfeat=1 but for some 
				# reason cloning not available?

				# if source image is not ZVOl, use dd method for cloning
				[ ${_cloud_source_is_zvol} -eq 0 ] && _cloud_truncate=1

				if [ ${_cloud_truncate} -eq 1 ]; then
					[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Clone cloud image into first/system vm disk (${W1_COLOR}dd${N1_COLOR} method)${N0_COLOR}"
					# to generic clonedata (add dd method)?
					# Linux does not support postfix in bs=, e.g. bs=4m
					case "${platform}" in
						Linux)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4000000 | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
						*)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4m | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
					esac
					echo
					# adjust original image size
					${TRUNCATE_CMD} -s${_dsk_ebytes} ${data}/dsk1.vhd
				else
					[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Clone cloud image into first/system vm disk (zfs clone method)${N0_COLOR}"
					# to generic clonedata ?
						_cloud_snapshot_name="${_cloud_source_zvol}@boot-${jname}"
						echo "${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name}" 2>&1
						${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name} > /dev/null 2>&1
						_ret=$?
						if [ ${_ret} -eq 1 ]; then
							# create cloud snapshot for $jname
							${ZFS_CMD} snapshot ${_cloud_source_zvol}@boot-${jname}
							# destory original zvol disk for vm
							${ZFS_CMD} destroy ${_dsk_source_zvol}
							#${ZFS_CMD} clone ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							${ZFS_CMD} clone -o volsize=${_dsk_ebytes} ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							cbsdlogger NOTICE ${CBSD_APP}: zfs clone method cloud image: gpart commit zvol/${_dsk_source_zvol}
							${GPART_CMD} commit zvol/${_dsk_source_zvol}
							# restore original size
							# ${ZFS_CMD} set volsize=${_dsk_ebytes} ${_dsk_source_zvol}
							# bug here, need for atomic ops ^^ in clone action
							${ZFS_CMD} set cbsdsnap:jname=${jname} ${_cloud_source_zvol}@boot-${jname}
							${ZFS_CMD} set cbsdsnap:snapname=cloud ${_cloud_source_zvol}@boot-${jname}
						else
							err 1 "${N1_COLOR}snapshot already exist ${_cloud_snapshot_name}${N0_COLOR}"
						fi
				fi
				echo "Eject cloud source: media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname}" 2>&1
				media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname} quiet=${quiet} 2>&1
			elif [ "${iso2img}" = "1" ]; then
				# Copy ISO as disk ?
				# Some profiles may use this, for example: Linux Tails
				local _orig_vm_iso_path="${vm_iso_path}"
				vm_boot="cd"
				init_iso
				if [ $? -eq 1 ]; then
					err 1 "${N1_COLOR}No such ISO source: ${N2_COLOR}${vm_iso_path}/${iso_img}${N0_COLOR}"
				fi
				vm_boot="hdd"
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ISO image initialization..${N0_COLOR}"
				local _myfile="${iso_img}"
				[ ! -r "${_myfile}" ] && err 1 "${N1_COLOR}Error: bstart: not readable: ${N2_COLOR}${_myfile}${N0_COLOR}"

				. ${subrdir}/zfs.subr

				if is_getzvol ${_myfile}; then
					_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_cloud_source_is_zvol=1
					_cloud_source_zvol="${is_zvol}"
				else
					_cloud_source_is_zvol=0
					_cloud_source_zvol=
					_ebytes=$( get_file_bytes ${_myfile} )
				fi

				# store original disk size to restore them after replace by cloud image
				if is_getzvol ${data}/dsk1.vhd; then
					_dsk_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_dsk_source_zvol="${is_zvol}"
					_cloud_truncate=0
				else
					_dsk_ebytes=$( get_file_bytes ${data}/dsk1.vhd )
					_cloud_truncate=1
					_dsk_source_zvol=
				fi

				# print some warning about not optimal when zfsfeat=1 but for some 
				# reason cloning not available?

				# if source image is not ZVOl, use dd method for cloning
				[ ${_cloud_source_is_zvol} -eq 0 ] && _cloud_truncate=1

				if [ ${_cloud_truncate} -eq 1 ]; then
					[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ISO into first/system vm disk (${W1_COLOR}dd${N1_COLOR} method)${N0_COLOR}"
					# to generic clonedata (add dd method)?
					# Linux does not support postfix in bs=, e.g. bs=4m
					case "${platform}" in
						Linux)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4000000 | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
						*)
							${NICE_CMD} -n 20 ${DD_CMD} if=${_myfile} bs=4m | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
							;;
					esac
					echo
					# adjust original image size
					${TRUNCATE_CMD} -s${_dsk_ebytes} ${data}/dsk1.vhd
				else
					[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ISO into first/system vm disk (zfs clone method)${N0_COLOR}"
					# to generic clonedata ?
						_cloud_snapshot_name="${_cloud_source_zvol}@boot-${jname}"
						echo "${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name}" 2>&1
						${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name} > /dev/null 2>&1
						_ret=$?
						if [ ${_ret} -eq 1 ]; then
							# create cloud snapshot for $jname
							${ZFS_CMD} snapshot ${_cloud_source_zvol}@boot-${jname}
							# destory original zvol disk for vm
							${ZFS_CMD} destroy ${_dsk_source_zvol}
							#${ZFS_CMD} clone ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							${ZFS_CMD} clone -o volsize=${_dsk_ebytes} ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							cbsdlogger NOTICE ${CBSD_APP}: zfs clone method cloud image: gpart commit zvol/${_dsk_source_zvol}
							${GPART_CMD} commit zvol/${_dsk_source_zvol}
							# restore original size
							# ${ZFS_CMD} set volsize=${_dsk_ebytes} ${_dsk_source_zvol}
							# bug here, need for atomic ops ^^ in clone action
							${ZFS_CMD} set cbsdsnap:jname=${jname} ${_cloud_source_zvol}@boot-${jname}
							${ZFS_CMD} set cbsdsnap:snapname=cloud ${_cloud_source_zvol}@boot-${jname}
						else
							err 1 "${N1_COLOR}snapshot already exist ${_cloud_snapshot_name}${N0_COLOR}"
						fi
				fi
				echo "Eject cloud source: media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname}" 2>&1
				media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname} quiet=${quiet} 2>&1
			fi
		fi
	fi

	case "${vm_os_type}" in
		freebsd)
			if [ "${from_jail}" = "1" ]; then
				create_from_jail
			else
				# default mode
				manage_boot_by_empty_hdd
				if [ "${vm_boot}" = "cd" ]; then
					init_iso
					if [ $? -eq 1 ]; then
						printf "${N1_COLOR}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${N0_COLOR}"
						for i in $( ${JOT_CMD} 5 ); do
							printf "."
							sleep 1
						done
						echo
					fi
				fi

				case "${vm_efi}" in
					uefi_csm|uefi)
						bhyveload_cmd=""
						;;
					*)
						if [ "${vm_boot}" = "cd" ]; then
							bhyveload_cmd="${LOCKF_CMD} -s -t0 ${tmpdir}/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -d ${iso_img} -m ${vm_ram} ${jname}"
							bhyveload_cmd_once="${LOCKF_CMD} -s -t0 ${tmpdir}/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -m ${vm_ram} -d ${data}/${MDFILE} ${jname}"
						else
							bhyveload_cmd="${LOCKF_CMD} -s -t0 ${tmpdir}/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -m ${vm_ram} -d ${data}/${MDFILE} ${jname}"
						fi
						;;
				esac
			fi
			;;
		linux|other|windows)
			# profile
			readconf vm-${vm_os_type}-${vm_os_profile}.conf

			[ -z "${vm_profile}" ] && err 1 "${N1_COLOR}No such profile: ${N2_COLOR}vm-${vm_os_type}-${vm_os_profile}.conf${N0_COLOR}"
			# re-read jail params and apply personal after profile 
			[ "${lm}" = "0" ] && . ${subrdir}/rcconf.subr
			manage_boot_by_empty_hdd

			if [ "${vm_boot}" = "cd" ]; then
				init_iso
				if [ $? -eq 1 ]; then
					printf "${N1_COLOR}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${N0_COLOR}"
					for i in $( ${JOT_CMD} 5 ); do
						printf "."
						sleep 1
					done
					echo
				fi
			fi
			bhyveload_cmd=""
			;;

		openbsd|dflybsd|netbsd)
			# profile
			readconf vm-${vm_os_type}-${vm_os_profile}.conf
			[ -z "${vm_profile}" ] && err 1 "${N1_COLOR}No such profile: ${N2_COLOR}vm-${vm_os_type}-${vm_os_profile}.conf${N0_COLOR}"
			# re-read jail params and apply personal after profile

			if [ "${lm}" = "0" ]; then
				. ${subrdir}/rcconf.subr
			else
				. ${subrdir}/rrcconf.subr
			fi

			manage_boot_by_empty_hdd
			if [ "${vm_boot}" = "cd" ]; then
				init_iso
				if [ $? -eq 1 ]; then
					printf "${N1_COLOR}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${N0_COLOR}"
					for i in $( ${JOT_CMD} 5 ); do
						printf "."
						sleep 1
					done
					echo
				fi
			fi

			if [ "${vm_boot}" = "hdd" ]; then
				if [ -z "${MDFILE}" ]; then
					[ ${quiet} -ne 1 ] && ${ECHO} "${H5_COLOR}Notice: ${N1_COLOR}no hard disk, boot from CD...${N0_COLOR}"
					vm_boot="cd"
				else
					_err=$( ${FILE_CMD} -s ${data}/${MDFILE} | ${CUT_CMD} -d":" -f2 | ${XARGS_CMD} )
					# live migration todo: skipp this >> ?
					if [ "${_err}" = "data" ]; then
						[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Looks like ${N2_COLOR}${data}/${MDFILE}${N1_COLOR} is empty.${N0_COLOR}"
						if getyesno "May be you want to boot from CD?"; then
							vm_boot="cd"
						fi
					fi
				fi
			fi

			bhyveload_cmd=""
			;;
		*)
			err 1 "${N1_COLOR}Unknown vm profile: ${N2_COLOR}${vm_os_type}${N0_COLOR}"
	esac

	# for vnet we can make another action
	. ${subrdir}/vnet.subr

	#unset zero-value
	[ "${bhyve_flags}" = "0" ] && unset bhyve_flags
	[ "${vm_os_profile}" = "0" ] && unset vm_os_profile

	# reset global bhyve_pci_id_busy_list (used by compile_ func via next_pci_id()
	# and remove buffer file with old $bhyve_pci_id_busy_list ( used by add_bhyve_pci_id_busy )
	bhyve_pci_id_busy_list=
	[ -f ${jailsysdir}/${jname}/bhyve_pciid ] && ${RM_CMD} -f ${jailsysdir}/${jname}/bhyve_pciid

	# mark modified field to FALSE in pcibus table to find unmodified entries for pciid cleanup
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE pcibus SET modified=false

	# truncate pcibus_run table
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite DELETE FROM pcibus_run

	# delete orphaned vale_ports if exists
	cbsdsqlrw local "DELETE FROM vale_ports WHERE jname=\"${jname}\""

	# init bhyve_cpus
	if ! compile_bhyve_cpus_args; then
		${ECHO} "${N1_COLOR}Unable to compile bhyve_cpus_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		bhyve_cpus="${vm_cpus}"
	fi

	# init hostbridge_args
	if ! compile_hostbridge_args; then
		${ECHO} "${N1_COLOR}No such hostbridge for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset hostbridge_args
	fi

	# Must be after hoster and before cd/hdd
	if ! compile_uefi_boot_args; then
		unset uefi_boot_args
	fi

	# Must be after hoster bridge to take 2-4 slot id
	case "${vm_boot}" in
		hdd|net)
			# init dsk_args first
			if ! compile_dsk_args; then
				${ECHO} "${N1_COLOR}No such disk for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
				unset dsk_args
				unset dsk_bootable
			fi

			# init cd_args
			if ! compile_cd_args; then
				unset cd_args
			fi
			;;
		cd)
			# init cd_args first
			if ! compile_cd_args; then
				unset cd_args
			fi

			# init dsk_args
			if ! compile_dsk_args; then
				${ECHO} "${N1_COLOR}No such disk for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
				unset dsk_args
				unset dsk_bootable
			fi
			;;
		*)
			err 1 "${N1_COLOR}Unknown vm_boot method: ${N2_COLOR}${vm_boot}${N0_COLOR}"
			;;
	esac

	if ! compile_dsk_controller_args; then
		unset dsk_controller_args
	fi

	# init nic_args
	if ! compile_nic_args ; then
		${ECHO} "${N1_COLOR}No such nic for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset nic_args
	fi

	if ! compile_nvme_args; then
		unset compile_nvme_args
	fi

	# init pci_passthru_args
	if ! compile_pci_passthru_args; then
		${ECHO} "${N1_COLOR}No such pci_passthru for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset pci_passthru_args
	fi

	# init console_args
	if ! compile_console_args; then
		${ECHO} "${N1_COLOR}No such console for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset console_args
	fi

	# init virtiornd_args
	if ! compile_virtiornd_args; then
		${ECHO} "${N1_COLOR}No such rnd for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset virtiornd_args
	fi

	# init VirtFS compile_virtio_9p_args
	if ! compile_virtio_9p_args; then
		${ECHO} "${N1_COLOR}No such 9p for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset virtio_9p_args
	fi

	# init soundhw_args= (hda)
	if ! compile_soundhw_args; then
		${ECHO} "${N1_COLOR}No such hda for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset soundhw_args
	fi

	# init efi_args
	if ! compile_efi_args; then
		${ECHO} "${N1_COLOR}No such efi for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset efi_args
	fi


	# init vnc_args
	if ! compile_vnc_args; then
		unset vnc_args
	else
		case "${bhyve_vnc_tcp_bind}" in
			127.0.0.1|0.0.0.0|::|\[::\]|::1|\[::1\])
				;;
			*)
				getnics-by-ip ip=${bhyve_vnc_tcp_bind} mode=strict
				_ret=$?
				if [ ${_ret} -ne 0 ]; then
					err 1 "${N1_COLOR}Unable to bind bhyve VNC, please check bhyve_vnc_tcp_bind settings: ${N2_COLOR}${bhyve_vnc_tcp_bind}${N0_COLOR}"
				fi
				;;
		esac
		# todo: check for port
	fi

	# init lpc_args: after VNC_ARGS: we must know the VNC port when cd_vnc_wait=1
	if ! compile_lpc_args; then
		${ECHO} "${N1_COLOR}No such lpc for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset lpc_args
	fi

	if [ "${lm}" = "1" -a -n "${lm_rnodename}"  ]; then
		[ -z "${lm_dport}" ] && lm_dport="0"
		# prepare live_migration_args and {jailsysdir}/${jname}/live_migration.conf config for live migration
		printf "${N1_COLOR}Compile live migration settings...${N0_COLOR}"
		if ! compile_live_migration_args -a ${lm_rnodename} -b ${nodename} -p ${lm_dport}; then
			unset live_migration_args
			[ "${lm}" = "1" ] && err 1 "${N1_COLOR}failed${N0_COLOR}"
		else
			printf "${N2_COLOR}done${N0_COLOR}"
		fi
	fi

	# Poehali!
	passthr=""

	# purge old/orphaned devices from pcibus table (which not modified by compile_args)
	_res=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT idx FROM pcibus WHERE modified=false 2>/dev/null )
	if [ -n "${_res}" ]; then
		cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT pcislot_name,pcislot_desc FROM pcibus WHERE modified=false 2>/dev/null | while read _name _desc; do
			# debug: echo "Prune for: [${_name} ${_desc}]"
			cbsdlogger NOTICE ${CBSD_APP}: pcibus map: purge for old/orphaned devices for ${jname}: [${_name} ${_desc}]
		done
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite DELETE FROM pcibus WHERE modified=false
	fi

	# extra validate we have no collision in PCI bus
	_res=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite 'SELECT pcislot_bus,COUNT(*) FROM pcibus_run GROUP BY pcislot_bus HAVING COUNT(*) > 1 LIMIT 1' 2>/dev/null )

	if [ -n "${_res}" ]; then
		sqllistdelimer=" "
		sqllist "${_res}" pcislot_bus pcislot_bus_count
		unset sqllistdelimer
		${ECHO} "${W1_COLOR}Error: ${N1_COLOR}PCI collision for bus: ${N2_COLOR}${pcislot_bus}${N1_COLOR}, count=${pcislot_bus_count}:${N0_COLOR}"
		cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT pcislot_name,pcislot_desc FROM pcibus_run WHERE pcislot_bus=${pcislot_bus}
		echo
		${ECHO} "${N1_COLOR}Please use bconfig to re-configure devices or use: ${N2_COLOR}cbsd bpcibus jname=${jname} mode=reset${N1_COLOR} to reset${N0_COLOR}"
		bcleanup jname="${jname}"
		exit 1
	fi

	# for init external hook variables
	geniplist ${ip4_addr}

	vm_logfile=$( ${MKTEMP_CMD} )

	# The parameters in local rc.conf file can overwrite the settings in the database
	[ -r ${jailsysdir}/${jname}/etc/rc.conf ] && . ${jailsysdir}/${jname}/etc/rc.conf

	[ -r "${jailsysdir}/${jname}/helpers/brctl.sqlite" ] && nice=$( cbsdsqlro ${jailsysdir}/${jname}/helpers/brctl.sqlite "SELECT cur FROM forms WHERE param=\"nice\" LIMIT 1" | ${AWK_CMD} '{printf $1}' )
	[ -z "${nice}" ] && nice="0"
	[ "${nice}" != "0" -a ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}bhyve renice: ${N2_COLOR}${nice}${N0_COLOR}"

	# restore overwrite debug_engine
	[ -n "${odebug_engine}" ] && debug_engine="${odebug_engine}"
	[ -n "${obhyverun_wrapper}" ] && bhyverun_wrapper="${bhyverun_wrapper}"

${CAT_CMD} > ${jailsysdir}/${jname}/bhyve.conf <<EOF
vm_boot='${vm_boot}'
bhyveload_cmd='${bhyveload_cmd}'
tmuxcmd='${tmuxcmd}'
jname='${jname}'

bhyve_flags='${bhyve_flags}'
# with topology
vm_cpus='${bhyve_cpus}'
vm_ram='${vm_ram}'
vm_efi='${vm_efi}'
vm_console='${vm_console}'
hostbridge_args='${hostbridge_args}'
passthr='${passthr}'
lpc_args='${lpc_args}'
pci_passthru_args='${pci_passthru_args}'
virtiornd_args='${virtiornd_args}'
nic_args='${nic_args}'
uefi_boot_args='${uefi_boot_args}'
dsk_args='${dsk_args}'
dsk_bootable='${dsk_bootable}'
dsk_controller_args='${dsk_controller_args}'
cd_args='${cd_args}'
cd_args2='${cd_args2}'
nvme_args='${nvme_args}'
#
efi_args='${efi_args}'
vnc_args='${vnc_args}'
console_args='${console_args}'
mytap='${mytap}'
mydsk='${mydsk}'
mycd='${mycd}'
cd_boot_once='${cd_boot_once}'
bhyveload_cmd_once='${bhyveload_cmd_once}'
console_nmdm='${console_nmdm}'
vm_logfile='${vm_logfile}'
vm_vnc_port='${vm_vnc_port}'

bhyve_generate_acpi='${bhyve_generate_acpi}'
bhyve_wire_memory='${bhyve_wire_memory}'
bhyve_rts_keeps_utc='${bhyve_rts_keeps_utc}'
bhyve_force_msi_irq='${bhyve_force_msi_irq}'
bhyve_x2apic_mode='${bhyve_x2apic_mode}'
bhyve_mptable_gen='${bhyve_mptable_gen}'
bhyve_ignore_msr_acc='${bhyve_ignore_msr_acc}'

cd_vnc_wait="${cd_vnc_wait}"
bhyve_vnc_resolution="${bhyve_vnc_resolution}"
bhyve_vnc_tcp_bind="${bhyve_vnc_tcp_bind}"
vnc_bind="${bhyve_vnc_tcp_bind}"

vnc_password="${vnc_password}"
virtio_9p_args="${virtio_9p_args}"
bhyve_vnc_vgaconf="${bhyve_vnc_vgaconf}"
bhyve_vnc_kbdlayout="${bhyve_vnc_kbdlayout}"
tpm="${tpm}"

media_auto_eject="${media_auto_eject}"

nice='${nice}'
live_migration_args='${live_migration_args}'
debug_engine='${debug_engine}'

on_poweroff='${on_poweroff}'
on_reboot='${on_reboot}'
on_crash='${on_crash}'

xhci='${xhci}'
tablet='${tablet}'
soundhw_args='${soundhw_args}'
bhyve_cmd='${bhyve_cmd}'
uuid='${uuid}'

nic_address_cmd='${nic_address_cmd}'

bhyverun_wrapper='${bhyverun_wrapper}'
EOF

	[ -z "${bhyverun_wrapper}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: empty mandatory param: ${N2_COLOR}bhyverun_wrapper${N0_COLOR}"
	[ ! -r "${bhyverun_wrapper}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: no such bhyverun_wrapper script: ${N2_COLOR}${bhyverun_wrapper}${N0_COLOR}"

	# export all tap bridges
	for i in ${mytap}; do
		T=
		eval T="\$mytap_${i}_bridge"
		[ -z "${T}" ] && break
		# vale use ':' in interface name:
		# mytap_vale1:files3_members: name contains characters not allowed in shell
		# cast ':' to '_' ?
		i=$( echo ${i} | ${TR_CMD} ":" "_" )
		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/bhyve.conf mytap_${i}_members="${T}" >/dev/null
	done

	# export all dsk path as dsk_X_path
	_dsk_id=0
	for i in ${mydsk}; do
		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/bhyve.conf dsk_${_dsk_id}_path="${i}" >/dev/null
		_dsk_id=$(( _dsk_id + 1 ))
	done

	# export all cd path as cd_X_path
	_dsk_id=0
	for i in ${mycd}; do
		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/bhyve.conf cd_${_dsk_id}_path="${i}" >/dev/null
		_dsk_id=$(( _dsk_id + 1 ))
	done

	# create a configuration only
	if [ -n "${cfg_only}" ]; then
		${MV_CMD} ${jailsysdir}/${jname}/bhyve.conf ${cfg_only}
		err 0 "${N1_COLOR}config created: ${N2_COLOR}${cfg_only}${N0_COLOR}"
	fi

	if [ "${vm_boot}" = "net" ]; then
		T=
		eval T="\$nic_hwaddr0"
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}vm_boot=net, primary network MAC: ${N2_COLOR}${T}${N0_COLOR}"
	fi

	checkpoint_args=

	if [ -n "${checkpoint}" ]; then
		CHECKPOINT_DIR="${jailsysdir}/${jname}/checkpoints"
		CHECKPOINT="${CHECKPOINT_DIR}/${checkpoint}.ckp"
		if [ -r ${CHECKPOINT} ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Checkpoint found, starting from: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
			checkpoint_args="-r ${CHECKPOINT}"
		else
			err 1 "${N1_COLOR}Checkpoint not found: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
		fi
	fi

	# this section should be BEFORE cloud-init seed generator
	if [ "${lm}" = "0" ]; then
		export_bhyve_data_for_external_hook
		external_exec_master_script "master_prestart.d"
	fi

	# todo: shared bhyve/xen virtual
	# cloud-init enabled?
	# this section should be AFTER master_prestart
	# cloud_init and CLOUD_FILES was initialized earlier

	if [ ${cloud_init} -eq 1 ]; then
		case "${ci_template}" in
			fire*)
				${CAT_CMD} ${jailsysdir}/${jname}/cloud-init/* > ${data}/dsk2.vhd
				${TRUNCATE_CMD} -s1m ${data}/dsk2.vhd
				;;
			homeass)
				. ${subrdir}/zfs.subr
				if is_getzvol ${data}/dsk1.vhd; then
					zvol_dsk0_dev=$( ${READLINK_CMD} ${data}/dsk1.vhd )
					zvol_dsk0=$( echo ${zvol_dsk0_dev} | ${SED_CMD} s#/dev/zvol/## )
					${ZFS_CMD} set volmode=full ${zvol_dsk0}
					${GPART_CMD} recover ${zvol_dsk0_dev}
				else
					zvol_dsk0_dev=$( ${MDCONFIG_CMD} -a -t vnode -f ${data}/dsk1.vhd )
					_ret=$?
					if [ ${_ret} -ne 0 ]; then
						err 1 "${N1_COLOR}${CBSD_APP}: homeass cloud-init: error: ${N2_COLOR}${MDCONFIG_CMD} -a -t vnode -f ${data}/dsk1.vhd${N0_COLOR}"
					fi
					${GPART_CMD} recover ${zvol_dsk0_dev}
					zvol_dsk0_dev="/dev/${zvol_dsk0_dev}"
				fi

				mount_dir=$( ${MKTEMP_CMD} -d )
				${MOUNT_CMD} -t msdosfs -o rw ${zvol_dsk0_dev}p1 ${mount_dir}
				_ret=$?
				if [ ${_ret} -ne 0 ]; then
					err 1 "${N1_COLOR}${CBSD_APP}: homeass cloud-init: error: ${N2_COLOR}${MOUNT_CMD} -t msdosfs -o rw ${zvol_dsk0_dev}p1 ${mount_dir}${N0_COLOR}"
				fi
				${MKDIR_CMD} -p ${mount_dir}/CONFIG/network
				${CP_CMD} -a ${jailsysdir}/${jname}/cloud-init/* ${mount_dir}/CONFIG/network/
				${LS_CMD} -l ${mount_dir}/CONFIG/network/
				${CAT_CMD} -n ${mount_dir}/CONFIG/network/*
				${CP_CMD} -HL ${workdir}/.ssh/id_rsa.pub ${mount_dir}/CONFIG/authorized_keys

				${UMOUNT_CMD} ${mount_dir}
				${RMDIR_CMD} ${mount_dir}
				case "${zvol_dsk0_dev}" in
					md*)
						${MDCONFIG_CMD} -d -u ${zvol_dsk0_dev}
						;;
				esac
				;;
			*)
				${RM_CMD} -f ${jailsysdir}/${jname}/seed.iso
				if [ "${vm_os_type}" = "windows" ]; then
					GENISOIMAGE_CMD=$( which /usr/local/bin/genisoimage )
					[ -z "${GENISOIMAGE_CMD}" ] && err 1 "${N1_COLOR}no such genisoimage tool. Windows cloud required it. Please install it first: ${N2_COLOR}pkg install -y sysutils/genisoimage${N0_COLOR}"
					${GENISOIMAGE_CMD} -joliet-long -R -V config-2 -o ${jailsysdir}/${jname}/seed.iso ${jailsysdir}/${jname}/cloud-init
				else
					if [ -n "${MAKEFS_CMD}" ]; then
						GENISOIMAGE_CMD="${MAKEFS_CMD}"
					else
						GENISOIMAGE_CMD=$( which makefs )
					fi
					[ ! -x "${GENISOIMAGE_CMD}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such makefs${N0_COLOR}"

					case "${ci_template}" in
						*)
							_label="cidata"
							;;
					esac

					${GENISOIMAGE_CMD} -t cd9660 -o label="${_label}" -o isolevel=2 -o rockridge -o publisher="CBSD" ${jailsysdir}/${jname}/seed.iso ${jailsysdir}/${jname}/cloud-init
					# see /usr/src/usr.sbin/makefs/cd9660/cd9660_strings.c + /usr/src/usr.sbin/makefs/cd9660.c (cd9660_valid_a_char)
					# why upper here, whereis spec?
					${SED_CMD} -i${SED_DELIMER}'' s:CIDATA:cidata: ${jailsysdir}/${jname}/seed.iso
				fi
				;;
		esac
	fi

	bstart_logfile="${jailsysdir}/${jname}/bstart.log"
	logfile_args="-l ${bstart_logfile}"

	case "${debug_engine}" in
		gdb|lldb)
			if [ "${lm}" = "1" ]; then
				# for live migration always run in tmux due to non-interactive commands from remote side
				[ -z "${tmuxcmd}" ] && err 1 "${N1_COLOR}bstart: no such tmux cmd${N0_COLOR}"
				env cbsd_workdir="${workdir}" ${tmuxcmd} -2 -u new -d "${bhyverun_wrapper} -c ${jailsysdir}/${jname}/bhyve.conf -g ${debug_engine} ${checkpoint_args} ${logfile_args}"
			else
				env cbsd_workdir="${workdir}" ${bhyverun_wrapper} -c ${jailsysdir}/${jname}/bhyve.conf -g ${debug_engine} ${checkpoint_args} ${logfile_args}
			fi
			;;
		*)
			[ -z "${tmuxcmd}" ] && err 1 "${N1_COLOR}bstart: no such tmux cmd${N0_COLOR}"
			env cbsd_workdir="${workdir}" ${tmuxcmd} -2 -u new-session -d "${bhyverun_wrapper} -c ${jailsysdir}/${jname}/bhyve.conf -e 1 ${checkpoint_args} ${logfile_args}"
			# ? when "daemon -f": got: Error return from kevent change + no network  ( or > /dev/null 2>&1 )
			#env cbsd_workdir="${workdir}" /usr/local/cbsd/misc/daemonize -E cbsd_workdir="${workdir}" ${bhyverun_wrapper} -c ${jailsysdir}/${jname}/bhyve.conf -e 1 ${checkpoint_args} ${logfile_args} > ${bstart_logfile} 2>&1
			;;
	esac

	if [ -n "${console_nmdm}" ]; then
		[ -z "${tmuxcmd}" ] && err 1 "${N1_COLOR}bstart: no such tmux cmd${N0_COLOR}"
		${tmuxcmd} select-window -t 0

		for i in ${console_nmdm}; do
			${tmuxcmd} new-window -t ${con} "cu -l ${i} -s 9600"
			con=$(( con + 1 ))
		done

		con=0
		for i in ${console_nmdm}; do
			if [ ${con} -eq 0 ]; then
				${tmuxcmd} split-window -v -p 96 -t 0 "cu -l ${i} -s 9600" # DEBUG string
			else
				${tmuxcmd} new-window -t 1 "cu -l ${i} -s 9600"
			fi
			con=$(( con + 1 ))
		done

		${tmuxcmd} select-window -t 0
	fi

	search_cmd="bhyve: ${jname}"
	strlen_search_cmd=$( strlen "${search_cmd}" )

	[ -n "${dsk_bootable}" -a ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Boot device: ${N2_COLOR}${dsk_bootable}${N0_COLOR}"

	if [ ${pid_wait} -eq 1 ]; then
		[ ${quiet} -ne 1 ] && printf "${N1_COLOR}Waiting for PID${N0_COLOR}"
		for i in $( ${SEQ_CMD} 10 ); do
			check_for_bhyve_process -j ${jname}
			[ -n "${vm_pid}" ] && break
			sleep 1
			[ ${quiet} -ne 1 ] && printf "."
		done
		echo
	fi

	[ -z "${vm_pid}" ] && vm_pid="0"

	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}PID: ${N2_COLOR}${vm_pid}${N0_COLOR}"

	if [ "${vm_pid}" = "0" ]; then
		[ -r ${bstart_logfile} ] && ${CAT_CMD} ${bstart_logfile}
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Please use for debug: ${N2_COLOR}${bhyverun_wrapper} -c ${jailsysdir}/${jname}/bhyve.conf${N0_COLOR}"
	fi

	if [ "${lm}" = "0" ]; then
		cbsdsqlrw local "UPDATE jails SET jid=\"${vm_pid}\" WHERE jname=\"${jname}\""

		# update state_time
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET state_time="(strftime('%s','now'))"

		# update state_time, local SQLite for back compatible
		cbsdsqlrw local UPDATE jails SET state_time="(strftime('%s','now'))" WHERE jname='${jname}'

		[ "${vm_efi}" = "none" -a ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Use ${N2_COLOR}cbsd blogin ${jname}${N1_COLOR} for attach to console${N0_COLOR}"
	fi

	[ "${lm}" = "0" ] && external_exec_master_script "master_poststart.d"

	# CBSD QUEUE
	if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
		[ -n "${cbsd_bhyve_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_bhyve_queue_name} id=${jname} cmd=bstart status=2 data_status=1 workdir="${workdir}"
	fi

	fwcounter
	expose mode=apply

	# first-run hack for MYbQT via SSH method.
	if [ -r ${jailsysdir}/${jname}/mybqt ]; then
		${RM_CMD} ${jailsysdir}/${jname}/mybqt
		nodeip-expose mode=create jname=${jname}
	fi

	return 0
}

# MAIN for multiple jails
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

TRAP=""
emulator="bhyve"	# for jname_is_multiple
# bhyve_list can be init in CBSDfile case
[ -z "${bhyve_list}" ] && jname_is_multiple

if [ -n "${bhyve_list}" ]; then
	TMP_JLIST="${bhyve_list}"
else
	TMP_JLIST="${jname}"
fi

JLIST=

# check for actual vm list in arg list
jail_num=0
jname=
for i in ${TMP_JLIST}; do
	exist=$( cbsdsqlro local SELECT jname FROM bhyve WHERE jname=\"${i}\" LIMIT 1 )
	if [ -n "${exist}" ]; then
		JLIST="${exist} ${JLIST}"
		if [ ${jail_num} -eq 0 ]; then
			jname="${i}"
		else
			# when > 1, jname unused below
			unset jname
		fi
		jail_num=$(( jail_num + 1 ))
	fi
done

# this is multiple list, split it by parallel jstart execution
if [ ${jail_num} -gt 1 ]; then
	cbsdlogger NOTICE ${CBSD_APP}: executing for multiple starting: ${JLIST}
	# multiple astart always non interactive
	export inter=0

	st_time=$( ${DATE_CMD} +%s )

	if [ -z "${parallel}" -o "${parallel}" = "0" ]; then
		for jname in ${JLIST}; do
			/usr/local/bin/cbsd bstart inter=0 jname=${jname}
		done
	else
		_seq=0
		wait_pid_files=

		for jname in ${JLIST}; do
			_boot_delay=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT boot_delay FROM settings LIMIT 1 )
			_pid_wait=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT pid_wait FROM settings LIMIT 1 )
			[ -z "${_pid_wait}" ] && _pid_wait=1

			if [ -z "${_boot_delay}" ]; then
				_boot_delay=0
			fi

			if [ ${_boot_delay} -eq 0 ]; then
				_boot_delay="${_seq}"
			fi

			env NOINTER=1 /usr/local/cbsd/misc/daemonize -E NOINTER=1 -p ${ftmpdir}/bstart.${jname}.$$ /usr/local/bin/cbsd bstart jname=${jname} delay=${_boot_delay}

			if [ ${_pid_wait} -eq 1 ]; then
				if [ -z "${wait_pid_files}" ]; then
					wait_pid_files="${ftmpdir}/bstart.${jname}.$$"
				else
					wait_pid_files="${wait_pid_files} ${ftmpdir}/bstart.${jname}.$$"
				fi
			fi

			_seq=$(( _seq + 1 ))
		done

		#lets save .pid file
		sleep 2

		# wait loop
		for i in ${wait_pid_files}; do
			[ ! -r ${i} ] && continue
			cbsd_pwait --pid=$( ${CAT_CMD} ${i} ) --timeout=${parallel}
		done

		wait_for_fpid -a start -t ${parallel}
	fi

	end_time=$( ${DATE_CMD} +%s )
	diff_time=$(( end_time - st_time ))
	diff_time=$( displaytime ${diff_time} )
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CBSD_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"
	cbsdlogger NOTICE ${CBSD_APP}: executing for multiple done in ${diff_time}: ${JLIST}
	err 0 "${N1_COLOR}Multiple bstart: ${N2_COLOR}done${N0_COLOR}"
fi

# MAIN
# single jname
[ -z "${jname}" ] && jname="${1}"
[ -z "${jname}" ] && err 1 "${N1_COLOR}Give me jail name${N0_COLOR}"

. ${distsharedir}/bhyve.conf		# only for for MYCOL variables: used in exports below

if [ "${lm}" = "0" ]; then
	. ${subrdir}/rcconf.subr
	if [ $? -eq 1 ]; then
		# remote start
		[ ${sqlreplica} -eq 0 ] && err 1 "${N1_COLOR}No such bhyve domain: ${N2_COLOR}${jname}${N0_COLOR}"
		remotenode=$( bwhereis ${jname} )
		[ -z "${remotenode}" ] && err 1 "${N1_COLOR}No such domain: ${N2_COLOR}${jname}${N0_COLOR}"
		for i in ${remotenode}; do
			if [ "${i}" = "${nodename}" ]; then
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote bstart: found on nodename ${N2_COLOR}${nodename}${N1_COLOR}. Skipped${N0_COLOR}"
				continue
			fi
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote bstart: ${N2_COLOR}${jname} ${N1_COLOR}on${N2_COLOR} ${i}${N0_COLOR}"
			rexe node=${i} cbsd bstart jname=${jname}
			if [ $? -eq 0 ]; then
				# updating state and put task for retrinv inventory
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Updating inventory...${N0_COLOR}"
				task autoflush=2 mode=new retrinv node=${i} tryoffline=1 data=db > /dev/null 2>&1
			fi
		done
		exit 0
	fi

	# mainly to smooth mass start in astart.
	[ -n "${odelay}" ] && delay="${odelay}"
	if [ ${delay} -ne 0 ]; then
		cbsdlogger NOTICE ${CBSD_APP}: delayed boot for ${jname} via delay= params: ${delay} sec.
		sleep ${delay}
	fi

	# start bhyve
	st_time=$( ${DATE_CMD} +%s )

	[ ! -d "${path}" ] && ${MKDIR_CMD} ${path}
	[ -z "${mnt_start}" ] && mnt_start="0"
	if [ "${mnt_start}" != "0" ]; then
		if [ ! -r "${mnt_start}" -o ! -x "${mnt_start}" ]; then
			err 1 "mnt_start script not exist or not executable: ${mnt_start}"
		fi
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Execute mnt_start script: ${N2_COLOR}${mnt_start}${N0_COLOR}..."
		# external mount, reset zfsfeat
		zfsfeat=0
		[ ! -d ${data} ] && ${MKDIR_CMD} -p ${data}
		[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -p ${jailsysdir}/${jname}
		${mnt_start} -d ${data} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}
		_ret=$?
		if [ ${_ret} -ne 0 ]; then
			err 1 "${W1_COLOR}error: ${N1_COLOR}mnt_start script failed: ${N2_COLOR}${mnt_start} ${jname}${N0_COLOR}"
		fi
	fi
else
	# Request for live migration start
	. ${nodes}
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Request for live migration...${N0_COLOR}"

	. ${subrdir}/rcconf.subr
	st_time=$( ${DATE_CMD} +%s )

	[ $? -eq 0 ] && err 1 "${N1_COLOR}This is local domain! Please unregister first: ${N2_COLOR}${jname}${N0_COLOR}"

	RHST=

	if [ -z "${lm_rnodename}" ]; then
		# detect remote hoster name by $jname in cluster
		RHST=$( bwhereis ${jname} | ${HEAD_CMD} -n1 )
		[ -z "${RHST}" ] && err 1 "${N1_COLOR}No such domain in the current CBSD cluster: ${N2_COLOR}${jname}${N0_COLOR}"
		lm_rnodename="${RHST}"	# for rjrcconf
	else
		RHST="${lm_rnodename}"
	fi
	# extra check
	_tmp=$( cbsdsqlro nodes SELECT ip FROM nodelist WHERE nodename=\"${lm_rnodename}\" )
	[ -z "${_tmp}" ] && err 1 "${N1_COLOR}No such node in the current CBSD cluster: ${N2_COLOR}${lm_rnodename}${N0_COLOR}"

	# todo: check for node idle and online, rexe?
	# todo2: extra check for shared storage
	shared_dir="jails-data jails-rcconf jails-system"

	[ ${quiet} -ne 1 ] && ${ECHO} "   ${H5_COLOR}* ${N1_COLOR}check for shared storage: ${N0_COLOR}"

	for i in ${shared_dir}; do
		printf "      ${H5_COLOR}* ${N2_COLOR}${i}: ${N0_COLOR}"
		check_for_shared_dir -d ${i} -n ${lm_rnodename} || err 1 "${N1_COLOR}directory id do not match, dir not shared: ${N2_COLOR}${i}${N1_COLOR} ?${N0_COLOR}"
		printf "${H3_COLOR}ok${N0_COLOR}"
		echo
	done

	emulator="bhyve"
	. ${subrdir}/rrcconf.subr
	[ $? -eq 1 ] && err 1 "${N1_COLOR}Request for live migration: no such VM in cluster: ${N2_COLOR}${jname}${N1_COLOR}. Perhaps you forget to: ${N2_COLOR}cbsd node mode=add${N1_COLOR} ?${N0_COLOR}"
fi


# cloud-init enabled?
# we need this section (besides section cloud-init section in start_bhyve() to allocate pci bus id via touching seed.iso
# since some helpers may work with runtime (e.g pcibus + cloudinit) config
if [ -d ${jailsysdir}/${jname}/cloud-init ]; then
	CLOUD_FILES=$( ${FIND_CMD} ${jailsysdir}/${jname}/cloud-init/ -mindepth 1 -type f | ${XARGS_CMD} )
	# gen seed only if files exist
	if [ -n "${CLOUD_FILES}" ]; then
		cloud_init=1
		[ ${quiet} -ne 1 ] && ${ECHO} "${H5_COLOR}cloud-init: ${H3_COLOR}enabled${N0_COLOR}"
		${TOUCH_CMD} ${jailsysdir}/${jname}/seed.iso
	else
		cloud_init=0
		unset CLOUD_FILES
	fi
else
	cloud_init=0
fi

# disable for a while
# jailed bhyve? chrooted?
#_jailed=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite "SELECT jailed FROM settings LIMIT 1" 2>/dev/null )
#[ -z "${_jailed}" ] && _jailed="0"
#_chrooted=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite "SELECT chrooted FROM settings LIMIT 1" 2>/dev/null )
#[ -z "${_chrooted}" ] && _chrooted="0"
_secured=
_jailed=0
_chrooted=0
#[ "${_jailed}" = "${_chrooted}" -a "${_jailed}" = "1" ] && err 1 "${N1_COLOR}${CBSD_APP}: chrooted=1 and jailed=1 is mutually exclusive. There can be only one${N2_COLOR}"

#[ "${_jailed}" != "0" ] && _secured="jail"
#[ "${_chrooted}" != "0" ] && _secured="chroot"

if [ -n "${_secured}" ]; then
	_jname=$( freejname default_jailname=J${jname} )
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${_secured}ed bhyve. create temporary jail: ${N2_COLOR}${_jname}${N0_COLOR}"
	${SED_CMD} -e "s:%%jname%%:${_jname}:g" \
	-e "s:%%workdir%%:${workdir}:g" \
	-e "s:%%domain%%#:my.domain:g" ${distdir}/share/jail.jconf > ${tmpdir}/jail.$$
	jcreate jconf=${tmpdir}/jail.$$
	${RM_CMD} -f ${tmpdir}/jail.$$
	exec b${_secured} jname=${_jname} bname=${jname}
fi

default_profile="bhyve-default-default.conf"
readconf ${default_profile}

init_bhyve

[ "${create_cbsdsystem_tap}" = "1" ] && init_systap

readconf vnc.conf
readconf bstart.conf

readconf ${default_profile}

if [ "${lm}" = "0" ]; then
	. ${subrdir}/rcconf.subr
	[ $? -eq 1 ] && err 1 "${N1_COLOR}No such domain: ${N2_COLOR}${jname}${N0_COLOR}"
	[ ${jid} -ne 0 ] && err 1 "${N1_COLOR}Jail ${jname} already running, jid: ${N2_COLOR}${jid}${N0_COLOR}"
	main_sqlite_local="${jailsysdir}/${jname}/local.sqlite"
	[ ${status} -eq 2 ] && err 1 "${N1_COLOR}Jail in slave mode. Please ${N2_COLOR}cbsd jswmode mode=master${N1_COLOR} first${N0_COLOR}"
	[ "${emulator}" != "bhyve" ] && err 1 "${N1_COLOR}Not bhyve mode${N0_COLOR}"
else
	. ${subrdir}/rrcconf.subr
	[ $? -eq 1 ] && err 1 "${N1_COLOR}No such domain: ${N2_COLOR}${jname}${N0_COLOR}"
	# change SQL local path to remote sqlite (exported by . ${subrdir}/rrcconf.subr)
	main_sqlite_local="${remote_local_sqlite}"
fi

# cleanup old artifacts before start new session
bcleanup jname="${jname}"

# snapshot.h:#define BHYVE_RUN_DIR "/var/run/bhyve/"
[ ! -d /var/run/bhyve ] && ${MKDIR_CMD} /var/run/bhyve

[ -z "${vm_ram}" -o -z "${vm_cpus}" -o -z "${vm_os_type}" ] && err 1 "${N1_COLOR}Parameter is mandatory: ${N2_COLOR}vm_ram, vm_cpus, vm_os_type${N0_COLOR}"
[ -z "${iso_auto_fetch}" ] && iso_auto_fetch=0
[ -z "${debug}" ] && debug=0

[ -n "${ointer}" ] && inter="${ointer}"
[ -n "${inter}" ] && export NOINTER=1

if [ ${vm_cpus} -gt ${ncpu} -a ${vm_cpus} -lt ${vm_cpus_max} ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Warning! Current node cpu: ${N2_COLOR}${ncpu}${N1_COLOR}, guest cpu: ${N2_COLOR}${vm_cpus}. ${N1_COLOR}Overcommitting vCPUs can hurt perfomance.${N0_COLOR}"
elif [ ${vm_cpus} -lt 1 -o ${vm_cpus} -gt ${vm_cpus_max} ]; then
	${ECHO} "${N1_COLOR}Valid number of guest CPUs within 1 - ${vm_cpus_max} range. Current vm_cpus: ${N2_COLOR}${vm_cpus}${N0_COLOR}"
	${ECHO} "${N1_COLOR}See ${N2_COLOR}vm_cpus_max=${N1_COLOR} params in ${N2_COLOR}/usr/local/cbsd/etc/defaults/bhyve-default-default.conf ${N1_COLOR}file.${N0_COLOR}"
	err 1 "${N1_COLOR}Please use ${N2_COLOR}~cbsd/etc/bhyve-default-default.conf${N1_COLOR} to overwrite.${N0_COLOR}"
fi


# hardcoded first disk path from SQL. Todo: mark bootable disk(s)
# todo: NVMe
# todo: Controller
MDFILE=$( cbsdsqlro ${main_sqlite_local} SELECT dsk_path FROM bhyvedsk WHERE jname=\"${jname}\" AND dsk_type=\"vhd\" LIMIT 1 2>/dev/null )
if [ -z "${MDFILE}" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${H5_COLOR}Notice: ${N1_COLOR}no any storage device found for this VM${N0_COLOR}"
else
	if [ ! -f "${data}/${MDFILE}" -a ! -h "${data}/${MDFILE}" ]; then
		${ECHO} "${N1_COLOR}No such ${data}/${MDFILE} but mdsize flags is not null.${N0_COLOR}"

		# if zfsfeat=1, try scan for zvol
		[ "${zfsfeat}" != "1" ] && break

		readconf zfs.conf
		. ${subrdir}/zfs.subr
		DATA=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} 2>/dev/null )

		[ -z "${DATA}" ] && break

		for lunname in $( ${SEQ_CMD} 0 16 ); do
			if [ -r /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ]; then				# back compatible for CBSD <= 12.0.14
				${LN_CMD} -sf /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ${data}/dsk${lunname}.vhd
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Found zvol and create symlink: ${data}/dsk${lunname}.vhd -> ${DATA}/bcbsd-${jname}-dsk${lunname}.vhd"
			fi
		done
	fi
fi

# export variables for external hooks
export jname=${jname}

for _i in ${JARG} ${MYCOL}; do
	T=
	eval T="\$$_i"
	export ${_i}="${T}"
done

# for local bhyve only
if [ "${lm}" = "0" ]; then
	# test for incorrect state
	if [ ${status} -eq 3 ]; then
		cbsdsqlrw local UPDATE jails SET maintenance=\"${comment}\" WHERE jname=\"${jname}\"
		comment="cbsdsqlro local SELECT maintenance FROM jails WHERE jname=\"${jname}\""
		if [ "${comment}" = "Stopping_VM" ]; then
			jswmode jname=${jname} mode=master comment='0'
		else
			${ECHO} "${N1_COLOR}Bhyve in maintenance: ${N2_COLOR}${comment}${N0_COLOR}"
			err 1 "${N1_COLOR}Please finish maintenance and switch mode via: ${N2_COLOR}jswmode jname=${jname} mode=master comment='0'${N0_COLOR}"
		fi
	fi
fi

start_bhyve

# API compat: ~cbsd/jails-system/info*.*
/usr/local/cbsd/misc/daemonize /usr/local/cbsd/tools/save-jail-info jname=${jname}

end_time=$( ${DATE_CMD} +%s )
diff_time=$(( end_time - st_time ))
diff_time=$( displaytime ${diff_time} )

if [ "${lm}" = "0" ]; then
	cbsdlogger NOTICE ${CBSD_APP}: bhyve domain ${jname} started in ${diff_time}
else
	cbsdlogger NOTICE ${CBSD_APP}: bhyve domain ${jname} started for live migration for source node ${lm_rnodename} in ${diff_time}
fi

[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CBSD_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"

if [ -n "${start_post_message}" -a ${quiet} -ne 1 ]; then
	${ECHO} "${N1_COLOR}post-start message for ${N2_COLOR}${jname}${N2_COLOR}"
	echo
	${ECHO} "${N1_COLOR}${start_post_message}${N0_COLOR}"
	echo
fi

exit 0
