#!/usr/local/bin/cbsd
#v13.0.11
MYARG=""
# should be in sync with run_bhyve() func: tools/up script
MYOPTARG="jconf inter removejconf"
# allow all bhyve settings
BHYVE_ARGS="relative_path jname path data rcconf host_hostname ip4_addr astart nic_hwaddr nic_ratelimit zfs_snapsrc runasap interface rctl_nice emulator imgsize imgtype vm_cpus vm_ram vm_os_type \
vm_efi iso_site iso_img register_iso_name register_iso_as vm_hostbridge bhyve_flags virtio_type vm_os_profile swapsize vm_iso_path vm_guestfs vm_vnc_port bhyve_generate_acpi bhyve_wire_memory \
bhyve_rts_keeps_utc bhyve_force_msi_irq bhyve_x2apic_mode bhyve_mptable_gen bhyve_ignore_msr_acc cd_vnc_wait bhyve_vnc_resolution bhyve_vnc_tcp_bind bhyve_vnc_vgaconf nic_driver vnc_password \
media_auto_eject vm_cpu_topology debug_engine xhci cd_boot_firmware jailed chrooted on_poweroff on_reboot on_crash is_cloud ci_jname ci_fqdn ci_template ci_interface ci_ip4_addr ci_gw4 \
ci_nameserver_address ci_nameserver_searchci_adjust_inteface_helper ci_user_add ci_user_pw_user ci_user_pw_root ci_user_pubkey ci_user_pubkey2 uuid ci_interface_mtu ci_ip4_addr2 ci_gw42 ci_interface_mtu2 \
interface2 ci_interface2 nic_flags nic2_flags bhyve_vnc_kbdlayout progress_state_file extra_profile_dir flavor vars_img tpm app_frontend bootrom com1 com2 com3 com4"
MYOPTARG="${MYOPTARG} ${BHYVE_ARGS}"

MYDESC="Create bhyve VM from config file or args"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

The bhyve VM is created according to configuration file generated by bconstruct-tui.
You can see this configuration file if you answer negatively to the
'Do you want to create jail immediately?' question at the end of the dialogue.

In addition, you can override all parameters via command line arguments.
Many parameters are set by default via profiles,
e.g: ~cbsd/etc/defaults/bhyve-default-default.conf which can be overwrited via
~cbsd/etc/bhyve-default-default.conf or personal profile.

When you use ZFS, you may want to adjust some properties (e.g. reservation/compression/..) by default
via ~cbsd/etc/zfs.conf ( defaults: ~cbsd/etc/defaults/zfs.conf )

You can create your own profile and specify it when creating the VM.

If you create the environment often and in large quantities, the CBSD
recommends using the CBSDfile (Vagrant style) instead of bcreate/bconstruct-tui.

To minimize the number of parameters that do not change (or individual per-hoster), 
these parameters are convenient to override through the ~cbsd/etc/bhyve-default-default.conf file.

For example, instead of:

 cbsd bcreate jname=myvm1 runasap=1 ci4_gw="10.0.0.1" ci_gw42="2a01:4f8:140:918b::1" interface="bridge2" default_ci_interface_mtu="1450" bhyve_vnc_tcp_bind="0.0.0.0" [...other params]

Set const values:

 cat >> ~cbsd/etc/bhyve-default-default.conf <<EOF
runasap=1
ci4_gw="10.0.0.1"
ci_gw42="2a01:4f8:140:918b::1"
interface="bridge2"
default_ci_interface_mtu="1450"
bhyve_vnc_tcp_bind="0.0.0.0"
EOF

And now these settings can be omited:

 cbsd bcreate jname=myvm1 [...other params]

You can add custom profile directories via extra_profile_dir= args or ~cbsd/etc/bhyve-default-default.conf

${H3_COLOR}General Options${N0_COLOR}:

 ${N2_COLOR}bhyve_vnc_tcp_bind=${N0_COLOR} - VNC bind, e.g: '127.0.0.1', '0.0.0.0', '::1'.
 ${N2_COLOR}extra_profile_dir${N0_COLOR}   - Extra directory to search VMs profiles.
 ${N2_COLOR}flavor${N0_COLOR}              - Use flavor (named group of vm_cpus/vm_ram/imgsize): see 'cbsd vm-packages';
 ${N2_COLOR}imgsize=${N0_COLOR}            - VM first/boot disk size, e.g.: '10g', '21474836480000'.
 ${N2_COLOR}inter=${N0_COLOR}              - 0 to prevent any questions and to accept answers by default.
 ${N2_COLOR}interface2=${N0_COLOR}         - <parent>, create VM with two interfaces, <parent> is uplink for nic2,
                       do not confuse with the ci_interface2 parameter.
 ${N2_COLOR}nic_flags=${N0_COLOR}          - '0' to disable. Pass additional flags for NIC, e.g.: 'private'.
 ${N2_COLOR}nic_flags2=${N0_COLOR}         - '0' to disable. Pass additional flags for NIC2, e.g.: 'private'.
 ${N2_COLOR}quiet=${N0_COLOR}              - 0,1: be quiet, dont output verbose message.
 ${N2_COLOR}removejconf=${N0_COLOR}        - 0,1: remove jconf after bcreate? 0 - don't remove.
 ${N2_COLOR}runasap=${N0_COLOR}            - 0,1: when 1 - run a VM immediately (atomic bcreate+bstart).
 ${N2_COLOR}vm_cpus=${N0_COLOR}            - VM CPUs cores, e.g.: '2'.
 ${N2_COLOR}vm_os_profile=${N0_COLOR}      - <name>: full config file is: vm-\${vm_os_type}-\${vm_os_profile}.conf, file
                       must be present in ~cbsd/etc/defaults/ or ~cbsd/etc/ directory.
 ${N2_COLOR}vm_os_type=${N0_COLOR}         - <name>: full config file is: vm-\${vm_os_type}-\${vm_os_profile}.conf, file
                       must be present in ~cbsd/etc/defaults/ or ~cbsd/etc/ directory.
 ${N2_COLOR}vm_ram=${N0_COLOR}             - VM RAM, e.g.: '1g', '2147483648'.
 ${N2_COLOR}zfs_snapsrc=${N0_COLOR}        - <name>: use ZFS snapshot as data source.

${H3_COLOR}Cloud-Init Options${N0_COLOR}:

 ${N2_COLOR}ci_gw4=${N0_COLOR}            - <ipv4> (cloud-init profile only): set IPv4 gateway for VM.
 ${N2_COLOR}ci_interface2=${N0_COLOR}     - configure second interface via cloud-init,
                      do not confuse with the interface2 parameter.
 ${N2_COLOR}ci_interface_mtu=${N0_COLOR}  - set MTU for NIC1, default: 1500.
 ${N2_COLOR}ci_interface_mtu2=${N0_COLOR} - set MTU for NIC2, default: 1500.
 ${N2_COLOR}ci_ip4_addr=${N0_COLOR}       - <ipv4> (cloud-init profile only): set IPv4 address for VM,
                      default is: DHCP. Can be: 'DHCPv6' or static IPv4/IPv6.
 ${N2_COLOR}ci_ip4_addr2=${N0_COLOR}      - <ipv4> (cloud-init profile only): set IPv4 address for VM
                      NIC2 (see also: ci_interface2,ci_gw42). Possible values same as ci_ip4_addr.
 ${N2_COLOR}ci_user_pubkey=${N0_COLOR}    - full/relative path to authorized_keys or may contain pubkey
                      string itself, e.g: ci_user_pubkey=\"ssh-ed25519 XXXXX root@my.domain\".
                      (cloud-init profile only): set authorized_keys file for cloud-init user for VM.
 ${N2_COLOR}ci_user_pubkey2=${N0_COLOR}   - secondary full/relative path to authorized_keys or may contain
                      pubkey string itself, e.g: ci_user_pubkey2=\"ssh-ed25519 XXXXX root@my.domain\".
                      (cloud-init profile only): set authorized_keys file for cloud-init user for VM.
 ${N2_COLOR}ci_user_pw_user=${N0_COLOR}   - set password for cloud-init user.
 ${N2_COLOR}ci_user_pw_root=${N0_COLOR}   - set password for 'root' user.

${H3_COLOR}LPC devices (LPC PCI-ISA bridge/TTY-class devices)${N0_COLOR}:

 ${N2_COLOR}bootrom=${N0_COLOR}           - LPC device, bootrom, default values: '/usr/local/cbsd/upgrade/patch/efi.fd'
 ${N2_COLOR}com1=${N0_COLOR}              - LPC device, COM1 serial, default: 'stdio', possible values:
                       '0' to disable, 'serial' to enable nmdm for serial port, 'tcp' to enable TCP server for serial port;
 ${N2_COLOR}com2=${N0_COLOR}              - LPC device, default '0', possible values:
                       '0' to disable, 'serial' to enable nmdm for serial port, 'tcp' to enable TCP server for serial port;
 ${N2_COLOR}com3=${N0_COLOR}              - LPC device, default '0', possible values:
                       '0' to disable, 'serial' to enable nmdm for serial port, 'tcp' to enable TCP server for serial port;
 ${N2_COLOR}com4=${N0_COLOR}              - LPC device, default '0', possible values:
                       '0' to disable, 'serial' to enable nmdm for serial port, 'tcp' to enable TCP server for serial port;
 ${N2_COLOR}tpm=${N0_COLOR}               - LPC device: TPM, use '0' to disable, '/dev/tpm*' (passthru) or 'new' to emulate;

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd bcreate jname=vm1 vm_os_type=linux vm_os_profile=Debian-x86-12 vm_ram=1g vm_cpus=1 runasap=1 imgsize=10g
 # cbsd bcreate jname=c1 vm_ram=4g vm_cpus=2 vm_os_type=freebsd vm_os_profile=cloud-FreeBSD-ufs-x64-14.2 imgsize=20g ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 com1=serial
 # cbsd bcreate jname=gateway flavor=small1 vm_os_type=linux vm_os_profile=cloud-Debian-x86-12 ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 ci_interface2=bridge2 ci_ip4_addr2=192.168.0.2 ci_gw42=192.168.0.1

${H3_COLOR}See also${N0_COLOR}:

 cbsd bconstruct-tui --help
 cbsd up --help
 cbsd get-profiles --help
 cbsd vm-packages --help
 cbsd jail2iso --help
 cat ~cbsd/etc/defaults/bhyve-default-default.conf
 cat ~cbsd/etc/defaults/zfs.conf

"

CBSDMODULE="bhyve"
EXTHELP="wf_bcreate"

. ${subrdir}/nc.subr

app_frontend=

nic_flags=
nic2_flags=
quiet=0

# temporary hack for second nic
interface2=
ci_ip4_addr2=
ci_gw42=
ci_interface_mtu2=
ci_interface2=

readconf buildworld.conf
readconf zfs.conf

oextra_profile_dir=
extra_profile_dir=

flavor=
oflavor=
quiet=0
oquiet=0

bootrom=
com1=
com2=
com3=
com4=
obootrom=
ocom1=
ocom2=
ocom3=
ocom4=

. ${subrdir}/universe.subr
. ${subrdir}/freebsd_world.subr
. ${subrdir}/bhyve.subr

progress_state_file=
. ${cbsdinit}

# if some of params specified via args, store them as temporary vars
for i in ${BHYVE_ARGS}; do
	unset o${i}
	eval "o${i}=\$$i"
done

readconf bhyve-default-default.conf

. ${system}
. ${mdtools}
. ${jfs}

if [ -z "${jconf}" ]; then
	jconf=$( ${MKTEMP_CMD} )
	export CBSD_INIT_SAVE2FILE=${jconf}		# save args and generate jconf
	[ -z "${removejconf}" ] && removejconf="0"  # do not delete jconf by default
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf} removejconf="${removejconf}" >/dev/null 2>&1
	. ${cbsdinit}
	unset CBSD_INIT_SAVE2FILE
fi

[ -z "${jconf}" -a -z "${jname}" ] && err 1 "${N1_COLOR}Please set for bcreate: ${N2_COLOR}${jconf}${N0_COLOR}"
[ -n "${removejconf}" ] && oremovejconf="${removejconf}"

really_create_base()
{
	if [ "${vm_os_type}" = "freebsd" -a "${from_jail}" = "1" ]; then
		case ${vm_os_profile} in
			"FreeBSD-bsdinstall-jail")
				export UNAME_r="${ver}-RELEASE"
				export DISTRIBUTIONS="kernel.txz base.txz"
				bsdinstall jail ${data}
				unset UNAME_r
				nobase=1
				;;
			*)
				nobase=0 # 0 for jail2iso
				init_target_arch
				init_basedir
				init_kerneldir
				get_base -v ${ver}
				get_kernel
				${ECHO} "${N1_COLOR}Stage1: ${N2_COLOR}jcreate...${N0_COLOR}"
				echo "jcreate jconf=${temprcconf} pkg_bootstrap=0"
				jcreate jconf=${temprcconf} pkg_bootstrap=0
				customskel
				;;
		esac
	fi
}

really_create_vm()
{
	local _res _msg _ret

	# do not create disk
	[ "${imgtype}" = "none" ] && return 0

	_res=$( substr --pos=0 --len=5 --str=${imgtype} )
	if [ "${_res}" = "/dev/" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: bcreate for ${jname}: imgtype = raw: ${imgtype}
		 [ ! -c "${imgtype}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: no such character device: ${N2_COLOR}${imgtype}${N0_COLOR}"
		oimgtype="raw"
		raw_path_dsk0="${imgtype}"
		return 0
	fi

	#test for imgtype
	case ${zfsfeat} in
			1)
				;;
			*)
				# force switch imgtype to md when no zfsfeat
				imgtype="md"
	esac

	if [ "${from_jail}" = "1" ]; then
		${ECHO} "${N1_COLOR}Stage2: ${N2_COLOR}jail2iso...${N0_COLOR}"
		[ -z "${swapsize}" ] && swapsize="0"

		mountbase -o "" -p "" -d "" -c "" -s ""
		jail2iso jname=${jname} nameserver=${jnameserver} ip4_addr=${ip4_addr} gw4=${gw4} dstname=${jname}.$$.img swapsize=${swapsize} freesize=${imgsize} dstdir=/tmp host_hostname="${host_hostname}" media=bhyve quiet=1 vm_guestfs=${vm_guestfs} efi=1
		# fromfile=${temprcconf} addmod=0

		jremove ${jname}
		create_fs ${data}

		#test for zfs mounted & mount if not
		case ${zfsfeat} in
			1)
				. ${subrdir}/zfs.subr
				zfsmnt ${data}
				[ $? -eq 2 ] && ${ZFS_CMD} mount "${ZPOOL}"
				;;
		esac

		/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} ver="${ver}" > /dev/null
		${MV_CMD} /tmp/${jname}.$$.img ${data}/${defdsk}
	else
		_msg=$( virtual_create_dsk -p ${data}/${defdsk} -s ${imgsize} -f 1 -t ${imgtype} 2>&1 )
		_res=$?
		if [ ${_res} -ne 0 ]; then
			bremove ${jname}
			log_err 1 "bcreate error: Couldn't create the image file. ${_msg}"
		fi
	fi
}


### MAIN
[ ! -f "${jconf}" ] && log_err 1 "${N1_COLOR}no such jconf file${N0_COLOR}";
st_time=$( ${DATE_CMD} +%s )
over="${ver}"
oarch="${arch}"
jconf=$( ${REALPATH_CMD} ${jconf} )

# several defaults
[ -z "${mnt_start}" ] && mnt_start="0"
[ -z "${mnt_stop}" ] && mnt_stop="0"

# make tmp jconf and work with it, insofar CBSD can
# change content of jconf in the course of their work
# we must leave the original file intact
jconf_tmp="${jconf}.tmp"
${CP_CMD} -a ${jconf} ${jconf_tmp}

if [ -z "${delpkglist}" ]; then
	delpkglist=0
else
	delpkglist=1
fi

temprcconf="${ftmpdir}/jcreate_jconf.$$"

# TRIM DOS CRLF
${CAT_CMD} ${jconf_tmp} | ${TR_CMD} -d \\r > ${temprcconf}

# read jname
. ${temprcconf}

[ -z "${jname}" ] && err 1 "${N1_COLOR}${CBSD_APP}: please set: ${N2_COLOR}jname=${N0_COLOR}"

# map flavor
[ -n "${oflavor}" ] && flavor="${oflavor}"
if [ -n "${flavor}" ]; then
	unset oimgsize imgsize ovm_ram vm_ram ovm_cpus vm_cpus
	flavors_available=$( cbsdsqlro ${dbdir}/local.sqlite "SELECT name FROM vmpackages" | ${SORT_CMD} | ${XARGS_CMD} )
	flavor_exist=0
	for i in ${flavors_available}; do
		[ "${flavor}" = "${i}" ] && flavor_exist=1 && break
	done

	[ ${flavor_exist} -eq 0 ] && log_err 1 "${N1_COLOR}${CBSD_APP}: no such flavor [${flavor}], available flavors: ${N2_COLOR}${flavors_available}${N0_COLOR}"

	_res=$( cbsdsqlro ${dbdir}/local.sqlite "SELECT pkg_vm_cpus,pkg_vm_ram,pkg_vm_disk FROM vmpackages WHERE name=\"${flavor}\" LIMIT 1" 2>/dev/null )
 
	[ -z "${_res}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get vm_cpus/vm_ram/imgsize for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	sqllistdelimer="|"
	sqllist "${_res}" ovm_cpus ovm_ram oimgsize
	unset sqllistdelimer
	${ECHO} "${N1_COLOR}${CBSD_APP}: flavor ${flavor} -> vm_cpus/vm_ram/imgsize: ${N2_COLOR}[${_res}]${N0_COLOR}"

	[ -z "${ovm_ram}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get 'vm_ram' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	[ -z "${oimgsize}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get 'imgsize' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	[ -z "${ovm_cpus}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get 'vm_cpus' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"

	vm_ram="${ovm_ram}"
	imgsize="${oimgsize}"
	vm_cpus="${ovm_cpus}"

	unset flavor oflavor flavor_exist flavors_available _res
fi

# if some of params specified via args, restore them from temporary vars
for i in ${BHYVE_ARGS}; do
	eval _mytest=\$o$i
	# Adjust some missed optional args
	if [ -z "${_mytest}" ]; then
		case "${i}" in
			emulator)
				_mytest="bhyve"
				;;
			host_hostname)
				_mytest="${jname}.my.domain"
				;;
			path)
				_mytest="${jaildir}/${jname}"
				;;
			data)
				_mytest="${jaildatadir}/${jname}-${jaildatapref}"
				;;
			rcconf)
				_mytest="${jailrcconfdir}/rc.conf_${jname}"
				;;
			vm_os_type)
				_mytest=$( echo "${_mytest}" | ${TR_CMD} '[:upper:]' '[:lower:]' )
				;;
			*)
				# skip unknown args
				continue
				;;
		esac
	fi
	/usr/local/cbsd/misc/cbsdsysrc -qf ${temprcconf} ${i}="${_mytest}" > /dev/null 2>&1
done

jstatus jname=${jname} > /dev/null 2>&1
[ $? -eq 0 ] || log_err 1 "${N1_COLOR}VM already exist: ${N2_COLOR}${jname}${N0_COLOR}"

vm_os_type=$( echo "${vm_os_type}" | ${TR_CMD} '[:upper:]' '[:lower:]' )
/usr/local/cbsd/misc/cbsdsysrc -qf ${temprcconf} vm_os_type="${vm_os_type}" > /dev/null 2>&1

# profile
[ -z "${vm_os_profile}" ] && log_err 1 "${N1_COLOR}No such vm_os_profile=${N0_COLOR}"
[ -z "${vm_os_type}" ] && log_err 1 "${N1_COLOR}No such vm_os_type=${N0_COLOR}"
template_profile=
template_profile="vm-${vm_os_type}-${vm_os_profile}.conf"

[ -n "${oextra_profile_dir}" ] && extra_profile_dir="${oextra_profile_dir}"

users_template_profile="${etcdir}/${template_profile}"

if [ ! -r ${etcdir}/defaults/${template_profile} -a ! -r ${etcdir}/${template_profile} ]; then
	[ -z "${extra_profile_dir}" ] && err 1 "${N1_COLOR}${CBSD_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
	if [ -d "${extra_profile_dir}" ]; then
		if [ -r ${extra_profile_dir}/${template_profile} ]; then
			users_template_profile="${etcdir}/${template_profile} ${extra_profile_dir}/${template_profile}"
		else
			err 1 "${N1_COLOR}${CBSD_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
		fi
	else
		${ECHO} "${W1_COLOR}${CBSD_APP} warning: ${N1_COLOR}no such extra_profile_dir: ${N2_COLOR}${extra_profile_dir}${N0_COLOR}" >&2
		err 1 "${N1_COLOR}${CBSD_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
	fi
fi

[ -z "${mnt_start}" ] && mnt_start="0"

[ -n "${ointer}" ] && inter="${ointer}"
[ "${inter}" = "0" ] && export NOINTER=1

if [ "${mnt_start}" != "0" ]; then
	if [ ! -r "${mnt_start}" -o ! -x "${mnt_start}" ]; then
		err 1 "mnt_start script not exist or not executable: ${mnt_start}"
	fi
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Execute mnt_start script: ${N2_COLOR}${mnt_start}${N0_COLOR}..."
	# external mount, reset zfsfeat
	zfsfeat=0
	[ ! -d "${data}" ] && ${MKDIR_CMD} -m 0770 ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${mnt_start} -d ${data} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}
	_ret=$?
	if [ ${_ret} -ne 0 ]; then
		err 1 "${W1_COLOR}error: ${N1_COLOR}mnt_start script failed: ${N2_COLOR}${mnt_start} -d ${data} -f ${jailfstabdir} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}${N0_COLOR}"
	fi
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${data} ${jailsysdir}/${jname}

	create_fs ${data}
	[ $? -ne 0 ] && err 1 "${N1_COLOR}create_fs failed${N0_COLOR}"
	vm_zfs_guid="0"
fi

if [ "${zfsfeat}" = "1" -a "${mnt_start}" = "0" ]; then
	readconf zfs.conf
	. ${subrdir}/zfs.subr
	DATA=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} )
	_msg=$( ${ZFS_CMD} create -o mountpoint=${workdir}/vm/${jname} ${DATA}/${jname} )
	_res=$?
	if [ ${_res} -ne 0 ]; then
		echo "${_msg}"
		exit ${_res}
	fi

	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${workdir}/vm/${jname}
	${CHMOD_CMD} 0770 ${workdir}/vm/${jname}

	vm_zfs_guid=$( ${ZFS_CMD} get -Ho value guid ${DATA}/${jname} 2>/dev/null )
	[ -z "${vm_zfs_guid}" ] && vm_zfs_guid="0"

	[ -z "${data}" ] && data="${jaildatadir}/${jname}-${jaildatapref}"
	olddata="${data}"
	data="${workdir}/vm/${jname}"
	/usr/local/cbsd/misc/cbsdsysrc -qf ${temprcconf} data="${data}" > /dev/null 2>&1
	${LN_CMD} -sf ${data} ${olddata}
	${LN_CMD} -sf ${data} ${jailsysdir}/${jname}
else
	# no mnt_start and no ZFS
	[ ! -d "${data}" ] && ${MKDIR_CMD} -m 0770 ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${data} ${jailsysdir}/${jname}
	vm_zfs_guid="0"
fi

. ${subrdir}/build.subr
. ${temprcconf}

case "${platform}" in
	Linux)
		conf_owner=$( ${STAT_CMD} -c "%u" ${jconf_tmp} )
		conf_group=$( ${STAT_CMD} -c "%g" ${jconf_tmp} )
		;;
	*)
		conf_owner=$( ${STAT_CMD} -f "%u" ${jconf_tmp} )
		conf_group=$( ${STAT_CMD} -f "%g" ${jconf_tmp} ) 
		;;
esac

${TRUNCATE_CMD} -s0 ${jconf_tmp}

# Merge with default and profile settings ( users_template_profile can be '${etcdir}/${template_profile}' + '${extra_profile_dir}/${template_profile}"
for merge_me in ${etcdir}/defaults/bhyve-default-default.conf ${etcdir}/defaults/${template_profile} ${etcdir}/bhyve-default-default.conf ${users_template_profile} ${temprcconf}; do
	[ ! -r "${merge_me}" ] && continue;
	tmp_merge=$( ${MKTEMP_CMD} )
	merge from=${jconf_tmp} to=${merge_me} out=${tmp_merge}
	[ -f ${tmp_merge} ] && ${MV_CMD} ${tmp_merge} ${jconf_tmp}
done

# make permission for group write
${CHOWN_CMD} ${conf_owner}:${conf_group} ${jconf_tmp}

. ${jconf_tmp}

# minimal config allows not to specify rcconf variable
# e.g: minimal config
# jname="freebsd1"
# imgsize="10g";
# vm_os_profile="FreeBSD-x64-12.0";
# vm_profile="FreeBSD-x64-12.0"
[ -z "${rcconf}" ] && rcconf="${jailrcconfdir}/rc.conf_${jname}"

[ -n "${oremovejconf}" ] && removejconf="${oremovejconf}"

if [ ${removejconf} = "1" ]; then
	trap "${RM_CMD} -f ${temprcconf} ${jconf} ${jconf_tmp}" HUP INT ABRT BUS TERM  EXIT
else
	trap "${RM_CMD} -f ${temprcconf} ${jconf_tmp}" HUP INT ABRT BUS TERM  EXIT
fi

[ -z "${jname}" ] && log_err 1 "${N1_COLOR}No such jname variable${N0_COLOR}"

if [ "${vm_os_type}" = "freebsd" -a "${from_jail}" = "1" ]; then
	# change emulator type for jcreate
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} emulator="jail" > /dev/null 2>&1

	if [ -n "${jprofile}" ]; then
		. ${subrdir}/settings-tui.subr
		if [ -r "${etcdir}/jail-freebsd-${jprofile}.conf" ]; then
			${ECHO} "${N1_COLOR}Use profile: ${N2_COLOR}${etcdir}/jail-freebsd-${jprofile}.conf${N0_COLOR}"
			merge_apply_profiles ${etcdir}/jail-freebsd-${jprofile}.conf ${jconf_tmp}
		elif [ -r "${etcdir}/defaults/jail-freebsd-${jprofile}.conf" ]; then
			${ECHO} "${N1_COLOR}Use profile: ${N2_COLOR}${etcdir}/defaults/jail-freebsd-${jprofile}.conf${N0_COLOR}"
			merge_apply_profiles ${etcdir}/defaults/jail-freebsd-${jprofile}.conf ${jconf_tmp}
		fi
	fi

	/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} emulator="bhyve" > /dev/null 2>&1
fi

# adjust some missed variabled
readconf vnc.conf
[ -z "${bhyve_vnc_resolution}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} bhyve_vnc_resolution="${default_vnc_width}x${default_vnc_height}"  > /dev/null 2>&1
[ -z "${bhyve_vnc_tcp_bind}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} bhyve_vnc_tcp_bind="${default_vnc_tcp_bind}"  > /dev/null 2>&1
[ -z "${bhyve_vnc_vgaconf}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} bhyve_vnc_vgaconf="${default_bhyve_vnc_vgaconf}"  > /dev/null 2>&1
[ -z "${bhyve_vnc_kbdlayout}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} bhyve_vnc_kbdlayout="${default_bhyve_vnc_kbdlayout}"  > /dev/null 2>&1
[ -z "${tpm}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} tpm="${default_tpm}"  > /dev/null 2>&1

if [ -z "${cd_vnc_wait}" ]; then
	case "${default_vnc_wait}" in
		auto|1)
			/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} cd_vnc_wait="1"  > /dev/null 2>&1
			;;
		*)
			/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} cd_vnc_wait="0"  > /dev/null 2>&1
			;;
	esac
fi

[ -z "${vnc_password}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} vnc_password="${default_vnc_password}" > /dev/null 2>&1

# re-read jail params and apply personal after profile
. ${jconf_tmp}

# extra check
# args deps:
#  - interface2 required for ci_interface2
#  - ci_interface2 required for ci_ip4_addr2
# when ci_interface2 not empty - inherit interface2 settings
[ -n "${oci_interface2}" -a "${oci_interface2}" != "0" ] && ci_interface2="${ci_interface2}"
[ -n "${ointerface2}" -a "${ointerface}" != "0" ] && interface2="${ointerface2}"
[ -n "${ci_ip4_addr2}" -a -z "${ci_interface2}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: ci_ip4_addr2 set, but ci_interface2 is empty${N0_COLOR}"
[ -z "${interface2}" -a -n "${ci_interface2}" ] && interface2="${ci_interface2}"

# apply pkglist from tpl_pkglist
[ -n "${tpl_pkglist}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} pkglist="${tpl_pkglist}" > /dev/null 2>&1

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_bhyve_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_bhyve_queue_name} id=${jname} cmd=bcreate vm_ram=${vm_ram} vm_cpus=${vm_cpus} vm_os_type=${vm_os_type} astart=${astart} protected=${protected} vnc_port='0' status=1
fi

# ip validate
if [ -n "${interface}" -a "${inteface}" != "0" ]; then
	### CHECK FOR IP ( 1 - check for interfaces) ####
	checkip ip=${ips} check=1 > /dev/null 2>&1
	case $? in
		0)
			log_err 1 "${N1_COLOR}Ip not in pool range${N0_COLOR}"
			;;
		1)	;;
		2)
			${ECHO} "${N1_COLOR}Warning:${N2_COLOR} Ip already exists in LAN${N0_COLOR}"
			;;
		*)
			log_err 1 "Unknown code from checkip"
			;;
	esac
fi


# imgsize validate
if [ -n "${imgsize_min}" -a -n "${imgsize}" ]; then

	if ! is_number "${imgsize_min}"; then
		# imgsize_min is number. assume value in bytes
		imgsize_min_bytes="${imgsize_min}"
	else
		imgsize_min_bytes=$( get_bytes ${imgsize_min} )
	fi

	[ -z "${imgsize_min_bytes}" -o "${imgsize_min_bytes}" = "0" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to convert imgsize to bytes: ${N2_COLOR}${imgsize_min}${N0_COLOR}"

	imgsize_min_human_mb=$(( imgsize_min_bytes / 1024 / 1024 ))

	if ! is_number "${imgsize}"; then
		# imgsize is number. assume value in bytes
		imgsize_bytes="${imgsize}"
	else
		imgsize_bytes=$( get_bytes ${imgsize} )
	fi

	imgsize_human_mb=$(( imgsize_bytes / 1024 / 1024 ))

	if [ ${imgsize_bytes} -lt ${imgsize_min_bytes} ]; then
		# not fatal?
#		if [ "${zfsfeat}" = "1" ]; then
#			${ZFS_CMD} destroy ${DATA}/${jname}
#		fi
#		err 1 "${N1_COLOR}imgsize too small: ${N2_COLOR}${imgsize_bytes} ( ${imgsize_human_mb}mb ) < ${imgsize_min_bytes} ( ${imgsize_min_human_mb}mb )${N0_COLOR}"
		${ECHO} "${W1_COLOR}imgsize too small: ${N2_COLOR}${imgsize_bytes} ( ${imgsize_human_mb}mb ) < ${imgsize_min_bytes} ( ${imgsize_min_human_mb}mb )${N0_COLOR}"
	fi
fi

# vm_ram validate
if [ -n "${vm_ram_min}" -a -n "${vm_ram}" ]; then

	if ! is_number "${vm_ram_min}"; then
		# vm_ram_min is number. assume value in bytes
		vm_ram_min_bytes="${vm_ram_min}"
	else
		vm_ram_min_bytes=$( get_bytes ${vm_ram_min} )
	fi

	vm_ram_min_human_mb=$(( vm_ram_min_bytes / 1024 / 1024 ))

	if ! is_number "${vm_ram}"; then
		# vm_ram is number. assume value in bytes
		vm_ram_bytes="${vm_ram}"
	else
		vm_ram_bytes=$( get_bytes ${vm_ram} )
	fi

	vm_ram_human_mb=$(( vm_ram_bytes / 1024 / 1024 ))

	if [ ${vm_ram_bytes} -lt ${vm_ram_min_bytes} ]; then
		if [ "${zfsfeat}" = "1" ]; then
			${ZFS_CMD} destroy ${DATA}/${jname}
		fi
		err 1 "${N1_COLOR}vm_ram too small: ${N2_COLOR}${vm_ram_bytes} ( ${vm_ram_human_mb}mb ) < ${vm_ram_min_bytes} ( ${vm_ram_min_human_mb}mb )${N0_COLOR}"
	fi
fi

defdsk="dsk1.vhd"
defnic="nic1.vhd"

if [ -z "${zfs_snapsrc}" ]; then
	really_create_base
	really_create_vm
	${CP_CMD} ${jconf_tmp} ${rcconf}
else
	if [ ! -d ${data} ]; then
		create_fs ${data}
		[ $? -ne 0 ] && err 1 "${N1_COLOR}create_fs failed${N0_COLOR}"
	fi
fi

[ ! -d "${data}" ] && log_err 1 "Can't create datadir ${data}"
[ ! -d ${jailfstabdir}  ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}

/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyvedsk.schema bhyvedsk
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-dskcontroller.schema bhyve_dskcontroller
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-nvme.schema bhyve_nvme
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyvenic.schema bhyvenic
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-p9shares.schema p9shares
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-pcibus.schema pcibus
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-pcibus.schema pcibus_run
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-cpu_topology.schema cpu_topology
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-settings.schema settings
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-soundhw.schema bhyve_soundhw
${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO bhyve_soundhw ( soundhw_play, soundhw_rec ) VALUES ( '/dev/dsp', '/dev/dsp' )"

if [ -n "${lpc_temp_sql}" ]; then
	[ ! -r "${lpc_temp_sql}" ] && err 1 "${N1_COLOR}${CBSD_APP}: lpc_temp_sql file not readable: ${N2_COLOR}${lpc_temp_sql}${N0_COLOR}"
	[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && /usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-lpc.schema lpc
	# dump and import lpc table
	_tmp_table_file=$( ${MKTEMP_CMD} )
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CBSD_APP}: import lpc data from: ${N2_COLOR}${lpc_temp_sql} ${N1_COLOR}via${N2_COLOR} ${_tmp_table_file}${N1_COLOR} dump${N0_COLOR}"
	${SQLITE3_CMD} ${lpc_temp_sql} .dump > ${_tmp_table_file}
	${SQLITE3_CMD} ${jailsysdir}/${jname}/local.sqlite < ${_tmp_table_file}
	${RM_CMD} -f ${_tmp_table_file} ${lpc_temp_sql} ${lpc_temp_sql}-shm ${lpc_temp_sql}-wal
else
	# keep in sync with bconstruct-tui
	/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-bhyve-lpc.schema lpc
	[ -n "${otpm}" ] && tpm="${otpm}"
	[ -n "${obootrom}" ] && bootrom="${obootrom}"
	[ -n "${ocom1}" ] && com1="${ocom1}"
	[ -n "${ocom2}" ] && com2="${ocom2}"
	[ -n "${ocom3}" ] && com3="${ocom3}"
	[ -n "${ocom4}" ] && com4="${ocom4}"
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO 'lpc' ( 'lpcslot_name', 'lpcslot_desc', 'lpcslot_value' ) VALUES ( 'bootrom', 'bootrom device', '${bootrom}' ), ( 'com1', 'com1 device', '${com1}' ), ( 'com2', 'com2 device', '${com2}' ), ( 'com3', 'com3 device', '${com3}' ), ( 'com4', 'com4 device', '${com4}' ), ( 'tpm', 'TPM device', '${tpm}' );"
fi

[ -n "${fstablocal}" -a -f "${fstablocal}" ] && ${CP_CMD} ${fstablocal} ${jailfstabdir}/${jailfstabpref}${jname}.local

${CP_CMD} ${jconf_tmp} ${rcconf}
. ${rcconf}

# Finnaly export to SQLite
jregister jname=${jname} mode=new
_res=$?

if [ ${_res} -ne 0 ]; then
	${ECHO}
	${ECHO} "${N1_COLOR}Creating ${jname} failed: ${N2_COLOR}cbsd jregister${N0_COLOR}"
	${ECHO} "${N1_COLOR}Please review bad config file: ${N2_COLOR}/tmp/rc.conf_${jname}${N0_COLOR}"
	${MV_CMD} ${rcconf} /tmp
	#cleanup
	[ -f "${mount_fstab}" ] && ${RM_CMD} -f ${mount_fstab}
	remove_data_dir ${data}
	exit 1
fi

# available in 13.2-PRERELEASE++
if [ ${freebsdhostversion} -gt 1301510 ]; then
	# copy BHYVE_UEFI_VARS.fd
	if [ -n "${vars_img}" -a -r "${srcdir}/iso/${vars_img}" ]; then
		${CP_CMD} "${srcdir}/iso/${vars_img}" ${jailsysdir}/${jname}/BHYVE_UEFI_VARS.fd
		${ECHO} "${N1_COLOR}Custom UEFI VARS file: ${N2_COLOR}${srcdir}/iso/${vars_img}${N0_COLOR}"
	elif [ -r ${distdir}/upgrade/patch/BHYVE_UEFI_VARS.fd ]; then
		${CP_CMD} ${distdir}/upgrade/patch/BHYVE_UEFI_VARS.fd ${jailsysdir}/${jname}/BHYVE_UEFI_VARS.fd
	fi
fi

echo
if [ "${vm_zfs_guid}" != "0" ]; then
	[ -z "${NOINTER}" ] && ${ECHO} "${N1_COLOR}Global VM ZFS guid: ${N2_COLOR}${vm_zfs_guid}${N0_COLOR}"
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET vm_zfs_guid=\"${vm_zfs_guid}\"
fi
if [ "${uuid}" = "0" ]; then
	uuid=$( ${UUIDGEN_CMD} )
	#[ -z "${NOINTER}" ] && ${ECHO} "${N1_COLOR}Global VM UUID: ${N2_COLOR}${uuid}${N0_COLOR}"
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET uuid=\"${uuid}\"
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} uuid="${uuid}" > /dev/null
fi

if [ -z "${NOINTER}" ]; then
	${ECHO} "${N1_COLOR}To edit VM properties use: ${N2_COLOR}cbsd bconfig jname=${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To start VM use: ${N2_COLOR}cbsd bstart ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To stop VM use: ${N2_COLOR}cbsd bstop ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To remove VM use: ${N2_COLOR}cbsd bremove ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}For attach VM console use: ${N2_COLOR}cbsd blogin ${jname}${N0_COLOR}"
	echo
	${ECHO} "${N1_COLOR}Creating ${jname} complete: ${N2_COLOR}Enjoy!${N0_COLOR}"
fi

${RM_CMD} -f ${rcconf}

[ -n "${oimgtype}" ] && imgtype="${oimgtype}"

# todo: move to dsk function
case "${imgtype}" in
	none)
		;;
	raw)
		# first disk path, todo: multidsk
		[ -z "${raw_path_dsk0}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: empty raw_path_dsk0 for raw type disk${N0_COLOR}"
		[ -z "${virtio_type}" ] && virtio_type="virtio-blk"
		_res=$( bhyve-dsk mode=attach jname=${jname} dsk_controller=${virtio_type} dsk_path=${raw_path_dsk0} imgtype=raw 2>&1 )
		_ret=$?
		[ ${_ret} -ne 0 ] && log_err 1 "${N1_COLOR}${CBSD_APP}: bhyve-dsk error:${N2_COLOR}${_res}${N0_COLOR}"
		;;
	*)
		if [ -z "${sectorsize}" ]; then
			sectorsize="${default_sectorsize}"
			[ -z "${sectorsize}" ] && sectorsize="4096"
		fi

		if [ ${zfsfeat} -eq 1 -a "${imgtype}" = "zvol" ]; then
			. ${subrdir}/zfs.subr
			dsk_zfs_guid=$( get_dsk_zfs_guid -p ${data}/${defdsk} 2>&1 )
			_ret=$?
			[ ${_ret} -ne 0 ] && dsk_zfs_guid="0"
		fi

		[ -z "${dsk_zfs_guid}" ] && dsk_zfs_guid="0"

		dsk_bsize=0

		if is_number "${imgsize}"; then
			if conv2bytes ${imgsize}; then
				dsk_bsize="${convval}"
			else
				dsk_bsize=0
			fi
		else
			# already on bytes ?
			dsk_bsize="${imgsize}"
		fi

		if [ -n "${virtio_type}" ]; then
			${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO bhyvedsk ( jname,dsk_controller,dsk_path,dsk_slot,dsk_size,dsk_sectorsize,dsk_zfs_guid ) VALUES ( \"${jname}\",\"${virtio_type}\",\"${defdsk}\","0",\"${dsk_bsize}\",\"${sectorsize}\", \"${dsk_zfs_guid}\" )"
		else
			${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO bhyvedsk ( jname,dsk_path,dsk_slot,dsk_size,dsk_sectorsize,dsk_zfs_guid ) VALUES ( \"${jname}\",\"${defdsk}\","0",\"${dsk_bsize}\",\"${sectorsize}\",\"${dsk_zfs_guid}\" )"
		fi

		# mark the first disk bootable
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET bootable='true' WHERE dsk_path=\"${defdsk}\""
		;;
esac

if [ -n "${nic_driver}" ]; then
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO bhyvenic ( jname,nic_driver,nic_parent ) VALUES ( \"${jname}\", \"${nic_driver}\", \"${interface}\" )"
else
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO bhyvenic ( jname,nic_parent ) VALUES ( \"${jname}\", \"${interface}\" )"
fi

if [ -n "${nic_flags}" ]; then
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvenic SET nic_flags=\"${nic_flags}\" WHERE nic_parent=\"${interface}\""
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} nic_flags="${nic_flags}" > /dev/null 2>&1
fi

if [ -n "${interface2}" -a "${interface2}" != "0" ]; then
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO bhyvenic ( jname,nic_parent ) VALUES ( \"${jname}\", \"${interface2}\" )"
	if [ -n "${ci_interface_mtu2}" ]; then
		${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvenic SET nic_mtu=\"${ci_interface_mtu2}\" WHERE nic_parent=\"${interface2}\""
	fi
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} interface2="${interface2}" > /dev/null 2>&1
	if [ -n "${nic2_flags}" ]; then
		${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvenic SET nic_flags=\"${nic2_flags}\" WHERE nic_parent=\"${interface2}\""
		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} nic_flags="${nic2_flags}" > /dev/null 2>&1
	fi
fi

[ -z "${nic_ratelimit}" ] && nic_ratelimit="0"

#if [ "${bhyve_have_net_ratelimit}" = "1" ]; then
#	if [ "${nic_ratelimit}" != "0" ]; then
#		cbsdlogger NOTICE ${CBSD_APP}: bcreate for ${jname}: nic_ratelimit for nic id 1: ${nic_ratelimit}
#		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvenic SET nic_ratelimit=\"${nic_ratelimit}\" WHERE id=\"1\""
#		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} nic_ratelimit="${nic_ratelimit}" > /dev/null 2>&1
#	fi
#fi

#[ -z "${dsk_iops_limit}" ] && dsk_iops_limit="0"
#[ -z "${dsk_mbps_limit}" ] && dsk_mbps_limit="0"

#if [ "${bhyve_have_dsk_ratelimit}" = "1" ]; then
#	if [ "${dsk_iops_limit}" != "0" ]; then
#		cbsdlogger NOTICE ${CBSD_APP}: bcreate for ${jname}: dsk_iops_limit for dsk ${defdsk}: ${dsk_iops_limit}
#		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET dsk_iops_limit=\"${dsk_iops_limit}\" WHERE dsk_path=\"${defdsk}\""
#		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} dsk_iops_limit="${dsk_iops_limit}" > /dev/null 2>&1
#	fi
#	if [ "${dsk_mbps_limit}" != "0" ]; then
#		cbsdlogger NOTICE ${CBSD_APP}: bcreate for ${jname}: dsk_mbps_limit for dsk ${defdsk}: ${dsk_mbps_limit}
#		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvedsk SET dsk_mbps_limit=\"${dsk_mbps_limit}\" WHERE dsk_path=\"${defdsk}\""
#		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} dsk_mbps_limit="${dsk_mbps_limit}" > /dev/null 2>&1
#	fi
#fi

# update state_time
cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET state_time="(strftime('%s','now'))"
cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET created="(strftime('%s','now'))"

# Check if SIZE if valid: can't be smaller then template
if is_number ${vm_ram}; then
	# not number, try to convert
	if conv2bytes "${vm_ram}"; then
		vm_ram="${convval}"
	else
		log_err 1 "${vm_ram} is not number and we can't convert int via conv2bytes"
	fi
fi

if [ -n "${register_iso_as}" -a -n "${register_iso_name}" ]; then
	cd_name="${register_iso_as}"
	cd_path="${srcdir}/iso/${register_iso_name}"

	cd_rec_num=$( cbsdsqlro storage_media SELECT COUNT\(path\) FROM media WHERE name=\"${cd_name}\" AND path=\"${cd_path}\" AND type=\"iso\" AND jname=\"-\" )

	if [ "${cd_rec_num}" = "0" ]; then
		# register new ISO with assignment to this VM
		_res=$( media mode=register name="${register_iso_as}" path="${srcdir}/iso/${register_iso_name}" type=iso jname=${jname} 2>&1 )
		_ret=$?
		[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}bcreate: media register error: ${_res}${N0_COLOR}"
	else
		# we have free/unassignent CD. link to this VM
		cbsdsqlrw storage_media "UPDATE media SET jname=\"${jname}\" WHERE jname=\"-\" AND type=\"iso\" AND name=\"${cd_name}\" AND path=\"${cd_path}\""
	fi
fi

if [ ! -d ${jailsysdir}/${jname} ]; then
	${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jailsysdir}/${jname}
fi

if [ -d "${jailsysskeldir}" ]; then
	# we have custom skeldir. copy
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Applying custom skel system dir template from: ${N2_COLOR}${jailsysskeldir}${N0_COLOR}"
	${RSYNC_CMD} -a ${jailsysskeldir}/ ${jailsysdir}/${jname}/
fi

[ ! -d ${jailsysdir}/${jname}/etc ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}/etc

system_dir="create.d \
facts.d \
master_create.d \
master_poststart.d \
master_poststop.d \
master_prestart.d \
master_prestop.d \
master_reboot.d \
remove.d \
start.d \
stop.d"

for i in ${system_dir}; do
	if [ -n "${systemskeldir}" -a -d "${systemskeldir}/${i}" ]; then
		[ ! -d ${jailsysdir}/${jname}/${i} ] && ${MKDIR_CMD} -m 0775 -p ${jailsysdir}/${jname}/${i}
		${RSYNC_CMD} -az ${systemskeldir}/${i}/ ${jailsysdir}/${jname}/${i}/
	else
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}qcreate: warning: no such dir: ${N2_COLOR}${systemskeldir}/${i}${N0_COLOR}"
		continue
	fi
	${CHOWN_CMD} -R ${cbsduser}:${cbsduser} ${jailsysdir}/${jname}/${i}
done

# is cloud-init-based ?
if [ ${is_cloud} -eq 1 ]; then
	${ECHO} "${N1_COLOR}auto-generate cloud-init settings: ${N2_COLOR}${jailsysdir}/${jname}/cloud-init${N0_COLOR}" 1>&2
	# auto adjust some missed settings
	if [ -z "${ci_interface}" ]; then
		ci_interface="${interface}"
		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_interface="${ci_interface}" > /dev/null 2>&1
	fi
	[ -z "${ci_jname}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_jname="${jname}" > /dev/null 2>&1
	[ -z "${ci_fqdn}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_fqdn="${host_hostname}" > /dev/null 2>&1

	if [ -z "${ci_ip4_addr}" ]; then
		${ECHO} "${W1_COLOR}warning: ${N1_COLOR}cloud-init based profile but ci_ip4_addr not set. Force to: ${N2_COLOR}DHCP${N0_COLOR}" 2>&1
		ci_ip4_addr="DHCP"
	fi
	if [ -z "${ci_gw4}" ]; then
		case "${ci_ip4_addr}" in
			[Rr][Ee][Aa][Ll][Dd][Hh][Cc][Pp])
				true
				;;
			*)
				${ECHO} "${W1_COLOR}warning: ${N1_COLOR}cloud-init based profile but ci_gw4 not set. VM without gateway?${N0_COLOR}" 2>&1
				;;
		esac
	fi

	OIFS="${IFS}"
	IFS=","

	normalize_ip4_addr=
	normalize_ci_ip4_addr=

	# normalize IP addresses
	for _pureip in ${ci_ip4_addr}; do
		IFS="${OIFS}"

		case "${_pureip}" in
			[Dd][Hh][Cc][Pp])
				_pureip=$( dhcpd 2>/dev/null )
				if [ $? -eq 2 ]; then
					cbsdlogger NOTICE ${CBSD_APP}: no free IP address for DHCP in nodeippool
					err 1 "${N1_COLOR}${CBSD_APP}: no free IP address for DHCP in nodeippool${N0_COLOR}"
				fi

				_pureip="${_pureip}/24"		# todo: hardcoded mask
				_mod=1
				;;
			[Dd][Hh][Cc][Pp][vV]6)
				_pureip=$( dhcpdv6 2>/dev/null )
				if [ $? -eq 2 ]; then
					cbsdlogger NOTICE ${CBSD_APP}: no free IP address for DHCPv6 in nodeip6pool
					err 1 "${N1_COLOR}${CBSD_APP}: no free IP address for DHCPv6 in nodeip6pool${N0_COLOR}"
				fi

				_pureip="${_pureip}/64"		# todo: hardcoded mask
				_mod=1
				;;
			[Rr][Ee][Aa][Ll][Dd][Hh][Cc][Pp])
				# helper test/chech?
				_mod=0
				;;
			*)
				_mod=0
				;;
		esac

		if [ ${_mod} -eq 1 ]; then
			# cast/modification

			# prefix mask set/
			strpos --str="${_pureip}" --search="/"
			_nomask=$?

			iptype ${_pureip}
			_inet=$?

			_prefix=

			case ${_inet} in
				1)
					# v4
					if [ ${_nomask} -eq 0 ]; then
						_prefix="24"
					else
						_prefix="${_pureip#*/}"
						if is_number "${_prefix}"; then
							err 1 "${N1_COLOR}${CBSD_APP}: prefix not a number: ${_prefix}: ${N2_COLOR}${_pureip}${N1_COLOR}"
						fi
					fi
					;;
				2)
					# v6
					if [ ${_nomask} -eq 0 ]; then
						_prefix="64"
					else
						_prefix="${_pureip#*/}"
						if is_number "${_prefix}"; then
							err 1 "${N1_COLOR}${CBSD_APP}: prefix not a number: ${_prefix}: ${N2_COLOR}${_pureip}${N1_COLOR}"
						fi
					fi
					;;
				*)
					err 1 "${N1_COLOR}${CBSD_APP}: unknown IP type: ${N2_COLOR}${_pureip}${N0_COLOR}"
					;;
			esac

			if [ -z "${normalize_ip4_addr}" ]; then
				normalize_ip4_addr="${IWM}"
			else
				normalize_ip4_addr="${normalize_ip4_addr},${IWM}"
			fi
			if [ -z "${normalize_ci_ip4_addr}" ]; then
				normalize_ci_ip4_addr="${IWM}/${_prefix}"
			else
				normalize_ci_ip4_addr="${normalize_ci_ip4_addr},${IWM}/${_prefix}"
			fi
		else
			if [ -z "${normalize_ci_ip4_addr}" ]; then
				normalize_ci_ip4_addr="${_pureip}"
			else
				normalize_ci_ip4_addr="${normalize_ci_ip4_addr},${_pureip}"
			fi
		fi

		IFS=","
	done

	IFS="${OIFS}"

	# update ip4_addr/ci_ip4_addr
	# REALDHCP flag?
	if [ -n "${normalize_ci_ip4_addr}" ]; then

		_ip4_addr=
		# strip mask for ip4_addr
		OIFS="${IFS}"
		IFS=","
		for _pureip in ${normalize_ci_ip4_addr}; do
			IFS="${OIFS}"
			iptype ${_pureip}
			if [ -z "${_ip4_addr}" ]; then
				_ip4_addr="${IWM}"
			else
				_ip4_addr="${_ip4_addr},${IWM}"
			fi
			IFS=","
		done
		IFS="${OIFS}"

		ip4_addr="${_ip4_addr}"
		ci_ip4_addr="${normalize_ci_ip4_addr}"

		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ip4_addr="${ip4_addr}" ci_ip4_addr="${normalize_ci_ip4_addr}" > /dev/null 2>&1
		bset jname="${jname}" ip4_addr="${_ip4_addr}" ci_ip4_addr="${normalize_ci_ip4_addr}" > /dev/null 2>&1
	fi

	[ -n "${ci_gw42}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_gw42="${ci_gw42}" > /dev/null 2>&1

	if [ -n "${ci_ip4_addr2}" ]; then
		#todo: normalize

		_ip4_addr=
		# strip mask for ip4_addr
		OIFS="${IFS}"
		IFS=","
		for _pureip in ${ci_ip4_addr2}; do
			IFS="${OIFS}"
			iptype ${_pureip}
			if [ -z "${_ip4_addr}" ]; then
				_ip4_addr="${IWM}"
			else
				_ip4_addr="${_ip4_addr},${IWM}"
			fi
			IFS=","
		done
		IFS="${OIFS}"

		ip4_addr="${ip4_addr},${_ip4_addr}"

		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ip4_addr="${ip4_addr}" ci_ip4_addr2="${ci_ip4_addr2}" > /dev/null 2>&1
		bset jname="${jname}" ip4_addr="${ip4_addr}" > /dev/null 2>&1
		normalize_ip4_addr="${normalize_ip4_addr},${ci_ip4_addr2}"
	fi

	readconf cloud-init.conf

	[ "${ci_interface2}" = "0" ] && unset ci_interface2
	[ "${ci_ip4_addr2}" = "0" ] && unset ci_ip4_addr2
	[ "${ci_gw4}" = "0" ] && unset ci_gw4

	[ -z "${ci_nameserver_address}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_nameserver_address="${default_ci_nameserver_address}" > /dev/null 2>&1
	[ -z "${ci_nameserver_search}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_nameserver_search="${default_ci_nameserver_search}" > /dev/null 2>&1
	[ -z "${ci_interface_name}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_interface_name="${default_ci_interface_name}" > /dev/null 2>&1
	[ -z "${ci_user_pw_user}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pw_user="${ci_user_pw_user}" > /dev/null 2>&1
	[ -z "${ci_user_pw_root}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pw_root="${ci_user_pw_root}" > /dev/null 2>&1
	[ -z "${ci_user_pw_root}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pw_root="${ci_user_pw_root}" > /dev/null 2>&1
	[ -z "${ci_user_pubkey}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pubkey="${ci_user_pubkey}" > /dev/null 2>&1
	[ -z "${ci_user_pubkey2}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pubkey2="${ci_user_pubkey2}" > /dev/null 2>&1
	[ -z "${ci_interface2}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_interface2="${ci_interface2}" > /dev/null 2>&1
	[ -z "${ci_ip4_addr2}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_ip4_addr2="${ci_ip4_addr2}" > /dev/null 2>&1
	[ -z "${ci_gw4}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_gw4="${ci_gw4}" > /dev/null 2>&1

	# GET MAC
	nic_hwaddr0=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT nic_hwaddr FROM bhyvenic LIMIT 1 )

	if [ "${nic_hwaddr0}" = "0" ]; then
		# gen MAC
		nic_hwaddr0=$( mac_gen 00:a0:98 )
		cbsdlogger NOTICE ${CBSD_APP}: bcreate for ${jname}: MAC address randomized and updated for nic id 1: ${nic_hwaddr0}
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvenic SET nic_hwaddr=\"${nic_hwaddr0}\" WHERE id=\"1\""
	fi

	[ -z "${ci_nic_hwaddr0}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_nic_hwaddr0="${nic_hwaddr0}" > /dev/null 2>&1

	#/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ip4_addr="${ip4_addr}" > /dev/null 2>&1

	case "${vm_os_type}" in
		windows)
			cloudengine="cloudinit-base"
			;;
		*)
			cloudengine="cloud-init"
			;;
	esac

	case "${ci_template}" in
		tiny-cloud)
			cloudengine="tiny-cloud"
			;;
		fire*)
			cloudengine="fire"
			;;
		homeass)
			cloudengine="homeass"
			;;
	esac

	cbsdlogger NOTICE ${CBSD_APP}: cloudinit mode=gen fromfile=${jailsysdir}/${jname}/rc.conf_${jname} jname=${jname} cloudengine="${cloudengine}"
	cloudinit mode=gen fromfile=${jailsysdir}/${jname}/rc.conf_${jname} jname=${jname} cloudengine="${cloudengine}"
fi


# unset vnc_password when vnc_password=0 (reserved)
[ "${vnc_password}" = "0" ] && cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE settings SET vnc_password=''"

# set default nice from rctl_nice
[ -n "${rctl_nice}" ] && nice="${rctl_nice}"

# rctl
. ${distsharedir}/rctl.conf
for i in ${RCTL} ${RCTL_EXTRA}; do
	_val=
	eval _val="\$rctl_${i}"
	if [ -n "${_val}" ]; then
		jrctl jname=${jname} mode=set ${i}=${_val} > /dev/null 2>&1 || /usr/bin/true
	fi
done

# store profile in jail system dir
storeconf vm-${vm_os_type}-${vm_os_profile}.conf ${jailsysdir}/${jname}/etc/vm-${vm_os_type}-${vm_os_profile}.conf

# create ascii rc.conf file for overwriting values
${CAT_CMD} > ${jailsysdir}/${jname}/etc/rc.conf <<EOF
# The parameters in this file can overwrite the settings from SQLite3 or global params
# e.g:
# bhyve_flags="-p 1:1 -p 2:2"
EOF

geniplist ${ip4_addr}		# for ipvX_first_public-like vars
. ${subrdir}/jcreate.subr
export_bhyve_data_for_external_hook
external_exec_master_script "master_create.d"

# API compat: ~cbsd/jails-system/info*.*
/usr/local/cbsd/misc/daemonize /usr/local/cbsd/tools/save-jail-info jname=${jname}

jcleanup jname=${jname}

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_bhyve_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_bhyve_queue_name} id="${jname}" cmd=bcreate status=2 data_status=0
fi

# expose
if [ -n "${app_frontend}" ]; then
	case "${app_frontend}" in
		mybqt)
			${TOUCH_CMD} ${jailsysdir}/${jname}/mybqt
			;;
		*)
			true
			;;
	esac
fi

end_time=$( ${DATE_CMD} +%s )
cbsdlogger NOTICE ${CBSD_APP}: vm ${jname} has been created in $(( end_time - st_time ))s

[ "${runasap}" = "1" ] && bstart inter=0 jname=${jname} progress_state_file=${progress_state_file}

if [ -n "${vm_post_message}" ]; then
	${ECHO} "${H3_COLOR} Profile post-create message:${N0_COLOR}"
	${ECHO} "${vm_post_message}"
	echo
fi

exit 0
