#!/usr/local/bin/cbsd
#v11.1.12
MYARG=""
# should be in sync with run_qemu() func: tools/up script
MYOPTARG="jconf inter removejconf"
# allow all qemu settings
QEMU_ARGS="relative_path jname path data rcconf host_hostname ip4_addr astart nic_hwaddr nic_ratelimit zfs_snapsrc runasap interface rctl_nice emulator imgsize imgtype vm_cpus vm_ram vm_os_type vm_efi \
iso_site iso_img register_iso_name register_iso_as vm_hostbridge qemu_flags virtio_type vm_os_profile swapsize vm_iso_path vm_guestfs vm_vnc_port qemu_generate_acpi qemu_wire_memory \
qemu_rts_keeps_utc qemu_force_msi_irq qemu_x2apic_mode qemu_mptable_gen qemu_ignore_msr_acc cd_vnc_wait qemu_vnc_resolution qemu_vnc_tcp_bind qemu_vnc_vgaconf nic_driver \
vnc_password media_auto_eject vm_cpu_topology debug_engine xhci cd_boot_firmware jailed chrooted on_poweroff on_reboot on_crash is_cloud ci_jname ci_fqdn ci_template ci_interface ci_ip4_addr \
ci_gw4 ci_nameserver_address ci_nameserver_searchci_adjust_inteface_helper ci_user_add ci_user_pw_user ci_user_pw_root ci_user_pubkey uuid ci_interface_mtu ci_ip4_addr2 ci_gw42 ci_interface_mtu2 \
interface2 ci_interface2 machine machine_accel vga usb tablet xhci flavor"
MYOPTARG="${MYOPTARG} ${QEMU_ARGS}"
MYDESC="Create QEMU domain from config file or args"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:


The QEMU VM is created according to configuration file generated by qconstruct-tui.
You can see this configuration file if you answer negatively to the
'Do you want to create jail immediately?' question at the end of the dialogue.

In addition, you can override all parameters via command line arguments.
Many parameters are set by default via profiles,
e.g: ~cbsd/etc/defaults/qemu-default-default.conf which can be overwrited via
~cbsd/etc/qemu-default-default.conf or personal profile.

When you use ZFS, you may want to adjust some properties (e.g. reservation/compression/..) by default
via ~cbsd/etc/zfs.conf ( defaults: ~cbsd/etc/defaults/zfs.conf )

You can create your own profile and specify it when creating the VM.

If you create the environment often and in large quantities, the CBSD
recommends using the CBSDfile (Vagrant style) instead of qcreate/qconstruct-tui.

To minimize the number of parameters that do not change (or individual per-hoster),
these parameters are convenient to override through the ~cbsd/etc/qemu-default-default.conf file.

For example, instead of:

 cbsd qcreate jname=myvm1 runasap=1 ci4_gw="10.0.0.1" ci_gw42="2a01:4f8:140:918b::1" interface="bridge2" default_ci_interface_mtu="1450" qemu_vnc_tcp_bind="0.0.0.0" [...other params]

Set const values:

 cat >> ~cbsd/etc/qemu-default-default.conf <<EOF
runasap=1
ci4_gw="10.0.0.1"
ci_gw42="2a01:4f8:140:918b::1"
interface="bridge2"
default_ci_interface_mtu="1450"
qemu_vnc_tcp_bind="0.0.0.0"
EOF

And now these settings can be omited:

 cbsd qcreate jname=myvm1 [...other params]

You can add custom profile directories via extra_profile_dir= args or ~cbsd/etc/qemu-default-default.conf

In order to use the SPICE instead of the VNC, please rebuild qemu-tools with SPICE support:

 make -C /usr/ports/sysutils/qemu-tools config
 make -C /usr/ports/sysutils/qemu-tools reinstall

To be able to boot the system into UEFI, you must install OVMF.fd firmware:

  for FreeBSD: pkg install edk2-qemu-x64 (for x86);
  for Linux: apt install ovmf;

Set const values:

 cat >> ~cbsd/etc/qemu-default-default.conf <<EOF
runasap=1
ci4_gw="10.0.0.1"
ci_gw42="2a01:4f8:140:918b::1"
interface="bridge2"
default_ci_interface_mtu="1450"
qemu_vnc_tcp_bind="0.0.0.0"
EOF

And now these settings can be omited:

 cbsd qcreate jname=myvm1 [...other params]

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}flavor${N0_COLOR}             - Use flavor (named group of vm_cpus/vm_ram/imgsize): see 'cbsd vm-packages';
 ${N2_COLOR}imgsize=${N0_COLOR}           - VM first/boot disk size, e.g.: '10g', '21474836480000'.
 ${N2_COLOR}inter=${N0_COLOR}             - 0 to prevent any questions and to accept answers by default.
 ${N2_COLOR}interface2=${N0_COLOR}        - <parent>, create VM with two interfaces, <parent> is uplink for nic2,
                      do not confuse with the ci_interface2 parameter.
 ${N2_COLOR}quiet=${N0_COLOR}             - 0,1: be quiet, dont output verbose message.
 ${N2_COLOR}removejconf=${N0_COLOR}       - 0,1: remove jconf after bcreate? 0 - don't remove.
 ${N2_COLOR}runasap=${N0_COLOR}           - 0,1: when 1 - run a VM immediately (atomic bcreate+bstart).
 ${N2_COLOR}vm_cpus=${N0_COLOR}           - VM CPUs cores, e.g.: '2'.
 ${N2_COLOR}vm_os_profile=${N0_COLOR}     - <name>: full config file is: vm-\${vm_os_type}-\${vm_os_profile}.conf, file
                      must be present in ~cbsd/etc/defaults/ or ~cbsd/etc/ directory.
 ${N2_COLOR}vm_os_type=${N0_COLOR}        - <name>: full config file is: vm-\${vm_os_type}-\${vm_os_profile}.conf, file
                      must be present in ~cbsd/etc/defaults/ or ~cbsd/etc/ directory.
 ${N2_COLOR}vm_ram=${N0_COLOR}            - VM RAM, e.g.: '1g', '2147483648'.
 ${N2_COLOR}zfs_snapsrc=${N0_COLOR}       - <name>: use ZFS snapshot as data source.

${H3_COLOR}Cloud-Init Options${N0_COLOR}:

 ${N2_COLOR}ci_gw4=${N0_COLOR}            - <ipv4> (cloud-init profile only): set IPv4 gateway for VM.
 ${N2_COLOR}ci_interface2=${N0_COLOR}     - configure second interface via cloud-init,
                      do not confuse with the interface2 parameter.
 ${N2_COLOR}ci_interface_mtu=${N0_COLOR}  - set MTU for NIC1, default: 1500.
 ${N2_COLOR}ci_interface_mtu2=${N0_COLOR} - set MTU for NIC2, default: 1500.
 ${N2_COLOR}ci_ip4_addr=${N0_COLOR}       - <ipv4> (cloud-init profile only): set IPv4 address for VM,
                      default is: DHCP. Can be: 'DHCPv6' or static IPv4/IPv6.
 ${N2_COLOR}ci_ip4_addr2=${N0_COLOR}      - <ipv4> (cloud-init profile only): set IPv4 address for VM
                      NIC2 (see also: ci_interface2,ci_gw42). Possible values same as ci_ip4_addr.
 ${N2_COLOR}ci_user_pubkey${N0_COLOR}     - full/relative path to authorized_keys or may contain pubkey
                      string itself, e.g: ci_user_pubkey=\"ssh-ed25519 XXXXX root@my.domain\".
                      (cloud-init profile only): set authorized_keys file for cloud-init user for VM.
 ${N2_COLOR}ci_user_pw_user=${N0_COLOR}   - set password for cloud-init user.
 ${N2_COLOR}ci_user_pw_root=${N0_COLOR}   - set password for 'root' user.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd qcreate jname=vm1 vm_os_type=linux vm_os_profile=CentOS-7-x86_64 vm_ram=1g vm_cpus=1 runasap=1 imgsize=10g
 # cbsd qcreate jname=c1 vm_ram=4g vm_cpus=2 vm_os_type=freebsd vm_os_profile=cloud-FreeBSD-ufs-x64-13.2 imgsize=20g ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3
 # cbsd qcreate jname=gateway flavor=small1 vm_os_type=linux vm_os_profile=cloud-Debian-x86-12 ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 ci_interface2=bridge2 ci_ip4_addr2=192.168.0.2 ci_gw42=192.168.0.1

# When qemu-system-aarch64/riscv installed, (Non-?)-native ARCH also possible:

 # cbsd qcreate jname=vm1 vm_os_type=linux vm_os_profile=Debian-aarch64-12 vm_ram=4g vm_cpus=1 imgsize=10g runasap=1
 # cbsd qcreate jname=vm1 vm_os_type=freebsd vm_os_profile=FreeBSD-aarch64-14.2 vm_ram=8g vm_cpus=8 imgsize=10g runasap=1 qemu_vnc_tcp_bind="0.0.0.0"

# (Non-?)-native aarch64 CLOUD images:

 # cbsd qcreate jname=vm1 flavor=small1 vm_os_type=linux vm_os_profile=cloud-Debian-aarch64-12 ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 runasap=1
 # cbsd qcreate jname=vm1 flavor=small1 vm_os_type=freebsd vm_os_profile=cloud-FreeBSD-ufs-aarch64-14.2 ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 runasap=1

${H3_COLOR}See also${N0_COLOR}:

 cbsd qconstruct-tui --help
 cbsd up --help
 cbsd get-profiles --help
 cbsd vm-packages --help
 cbsd jail2iso --help
 cat ~cbsd/etc/defaults/qemu-default-default.conf
 cat ~cbsd/etc/defaults/zfs.conf

"
CBSDMODULE="qemu"
EXTHELP="wf_qcreate"

. ${subrdir}/nc.subr

nic_flags=
nic2_flags=
quiet=0

# temporary hack for second nic
interface2=
ci_ip4_addr2=
ci_gw42=
ci_interface_mtu2=
ci_interface2=

readconf buildworld.conf
readconf zfs.conf

oextra_profile_dir=
extra_profile_dir=

flavor=
oflavor=
quiet=0
oquiet=0

. ${subrdir}/universe.subr
. ${subrdir}/freebsd_world.subr
. ${subrdir}/qemu.subr

progress_state_file=
. ${cbsdinit}

# if some of params specified via args, store them as temporary vars
for i in ${QEMU_ARGS}; do
	unset o${i}
	eval "o${i}=\$$i"
done

# todo: per-platform overwrite, e.g. QEMU on FreeBSD and QEMU on DFLY/NetBSD with alt. defaults
readconf qemu-default-default.conf

. ${system}
. ${mdtools}
. ${jfs}

if [ -z "${jconf}" ]; then
	jconf=$( ${MKTEMP_CMD} )
	export CBSD_INIT_SAVE2FILE=${jconf}		# save args and generate jconf
	[ -z "${removejconf}" ] && removejconf="0"	# do not delete jconf by default
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf} removejconf="${removejconf}" >/dev/null 2>&1
	. ${cbsdinit}
	unset CBSD_INIT_SAVE2FILE
fi

[ -z "${jconf}" -a -z "${jname}" ] && err 1 "${N1_COLOR}Please set for qcreate: ${N2_COLOR}${jconf}${N0_COLOR}"
[ -n "${removejconf}" ] && oremovejconf="${removejconf}"

really_create_base()
{
	if [ "${vm_os_type}" = "freebsd" -a "${from_jail}" = "1" ]; then
		case ${vm_os_profile} in
			"FreeBSD-bsdinstall-jail")
				export UNAME_r="${ver}-RELEASE"
				export DISTRIBUTIONS="kernel.txz base.txz"
				bsdinstall jail ${data}
				unset UNAME_r
				nobase=1
				;;
			*)
				nobase=0 # 0 for jail2iso
				init_target_arch
				init_basedir
				init_kerneldir
				get_base -v ${ver}
				get_kernel
				${ECHO} "${N1_COLOR}Stage1: ${N2_COLOR}jcreate...${N0_COLOR}"
				echo "jcreate jconf=${temprcconf} pkg_bootstrap=0"
				jcreate jconf=${temprcconf} pkg_bootstrap=0
				customskel
				;;
		esac
	fi
}

really_create_vm()
{
	local _res _msg

	# do not create disk
	[ "${imgtype}" = "none" ] && return 0

	_res=$( substr --pos=0 --len=5 --str=${imgtype} )
	if [ "${_res}" = "/dev/" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: bcreate for ${jname}: imgtype = raw: ${imgtype}
		 [ ! -c "${imgtype}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: no such character device: ${N2_COLOR}${imgtype}${N0_COLOR}"
		oimgtype="raw"
		raw_path_dsk0="${imgtype}"
		return 0
	fi

	#test for imgtype
	case ${zfsfeat} in
		1)
			;;
		*)
			# force switch imgtype to md when no zfsfeat
			imgtype="md"
	esac

	if [ "${from_jail}" = "1" ]; then
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Stage2: ${N2_COLOR}jail2iso...${N0_COLOR}"
		[ -z "${swapsize}" ] && swapsize="0"

		mountbase -o "" -p "" -d "" -c "" -s ""
		#echo "jail2iso jname=${jname} nameserver=${jnameserver} ip4_addr=${ip4_addr} gw4=${gw4} dstname=${jname}.$$.img swapsize=${swapsize} freesize=${imgsize} dstdir=/tmp host_hostname="${host_hostname}" media=qemu quiet=1 prunelist=0 nobase=${nobase} vm_guestfs=${vm_guestfs} fromfile=${temprcconf}"
		jail2iso jname=${jname} nameserver=${jnameserver} ip4_addr=${ip4_addr} gw4=${gw4} dstname=${jname}.$$.img swapsize=${swapsize} freesize=${imgsize} dstdir=/tmp host_hostname="${host_hostname}" media=qemu quiet=1 vm_guestfs=${vm_guestfs}
		# fromfile=${temprcconf} addmod=0

		jremove ${jname}
		create_fs ${data}

		#test for zfs mounted & mount if not
		case ${zfsfeat} in
			1)
				. ${subrdir}/zfs.subr
				zfsmnt ${data}
				[ $? -eq 2 ] && ${ZFS_CMD} mount "${ZPOOL}"
				;;
		esac

		/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf} ver="${ver}" > /dev/null
		${MV_CMD} /tmp/${jname}.$$.img ${data}/${defdsk}
	else
		_msg=$( virtual_create_dsk -p ${data}/${defdsk} -s ${imgsize} -f 1 -t ${imgtype} 2>&1 )
		_res=$?
		if [ ${_res} -ne 0 ]; then
			qremove ${jname}
			err 1 "qcreate error: Couldn't create the image file. ${_msg}"
		fi
	fi
}

### MAIN
[ ! -f "${jconf}" ] && err 1 "${N1_COLOR}no such jconf file${N0_COLOR}";
st_time=$( ${DATE_CMD} +%s )
over="${ver}"
oarch="${arch}"
jconf=$( ${REALPATH_CMD} ${jconf} )

# several defaults
[ -z "${mnt_start}" ] && mnt_start="0"
[ -z "${mnt_stop}" ] && mnt_stop="0"

# make tmp jconf and work with it, insofar CBSD can
# change content of jconf in the course of their work
# we must leave the original file intact
jconf_tmp="${jconf}.tmp"
${CP_CMD} -a ${jconf} ${jconf_tmp}

if [ -z "${delpkglist}" ]; then
	delpkglist=0
else
	delpkglist=1
fi

temprcconf="${ftmpdir}/jcreate_jconf.$$"

# TRIM DOS CRLF
${CAT_CMD} ${jconf} | ${TR_CMD} -d \\r > ${temprcconf}

# read jname
. ${temprcconf}

[ -z "${jname}" ] && err 1 "${N1_COLOR}${CBSD_APP}: please set: ${N2_COLOR}jname=${N0_COLOR}"

# map flavor
[ -n "${oflavor}" ] && flavor="${oflavor}"
if [ -n "${flavor}" ]; then
	unset oimgsize imgsize ovm_ram vm_ram ovm_cpus vm_cpus
	flavors_available=$( cbsdsqlro ${dbdir}/local.sqlite "SELECT name FROM vmpackages" | ${SORT_CMD} | ${XARGS_CMD} )
	flavor_exist=0
	for i in ${flavors_available}; do
		[ "${flavor}" = "${i}" ] && flavor_exist=1 && break
	done

	[ ${flavor_exist} -eq 0 ] && log_err 1 "${N1_COLOR}${CBSD_APP}: no such flavor [${flavor}], available flavors: ${N2_COLOR}${flavors_available}${N0_COLOR}"

	_res=$( cbsdsqlro ${dbdir}/local.sqlite "SELECT pkg_vm_cpus,pkg_vm_ram,pkg_vm_disk FROM vmpackages WHERE name=\"${flavor}\" LIMIT 1" 2>/dev/null )

	[ -z "${_res}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get vm_cpus/vm_ram/imgsize for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	sqllistdelimer="|"
	sqllist "${_res}" ovm_cpus ovm_ram oimgsize
	unset sqllistdelimer
	${ECHO} "${N1_COLOR}${CBSD_APP}: flavor ${flavor} -> vm_cpus/vm_ram/imgsize: ${N2_COLOR}[${_res}]${N0_COLOR}"

	[ -z "${ovm_ram}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get 'vm_ram' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	[ -z "${oimgsize}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get 'imgsize' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	[ -z "${ovm_cpus}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: unable to get 'vm_cpus' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"

	vm_ram="${ovm_ram}"
	imgsize="${oimgsize}"
	vm_cpus="${ovm_cpus}"

	unset flavor oflavor flavor_exist flavors_available _res
fi

# if some of params specified via args, restore them from temporary vars
for i in ${QEMU_ARGS}; do
	eval _mytest=\$o$i
	# Adjust some missed optional args
	if [ -z "${_mytest}" ]; then
		case "${i}" in
			emulator)
				_mytest="qemu"
				;;
			host_hostname)
				_mytest="${jname}.my.domain"
				;;
			path)
				_mytest="${jaildir}/${jname}"
				;;
			data)
				_mytest="${jaildatadir}/${jname}-${jaildatapref}"
				;;
			rcconf)
				_mytest="${jailrcconfdir}/rc.conf_${jname}"
				;;
			vm_os_type)
				_mytest=$( echo "${_mytest}" | ${TR_CMD} '[:upper:]' '[:lower:]' )
				;;
			*)
				# skip unknown args
				continue
				;;
		esac
	fi
	/usr/local/cbsd/misc/cbsdsysrc -qf ${temprcconf} ${i}="${_mytest}" > /dev/null 2>&1
done

jstatus jname=${jname} > /dev/null 2>&1
[ $? -eq 0 ] || log_err 1 "${N1_COLOR}VM already exist: ${N2_COLOR}${jname}${N0_COLOR}"

vm_os_type=$( echo "${vm_os_type}" | ${TR_CMD} '[:upper:]' '[:lower:]' )
/usr/local/cbsd/misc/cbsdsysrc -qf ${temprcconf} vm_os_type="${vm_os_type}" > /dev/null 2>&1

# profile
[ -z "${vm_os_profile}" ] && log_err 1 "${N1_COLOR}No such vm_os_profile=${N0_COLOR}"
[ -z "${vm_os_type}" ] && log_err 1 "${N1_COLOR}No such vm_os_type=${N0_COLOR}"
template_profile=
template_profile="vm-${vm_os_type}-${vm_os_profile}.conf"

[ -n "${oextra_profile_dir}" ] && extra_profile_dir="${oextra_profile_dir}"

users_template_profile="${etcdir}/${template_profile}"

if [ ! -r ${etcdir}/defaults/${template_profile} -a ! -r ${etcdir}/${template_profile} ]; then
	[ -z "${extra_profile_dir}" ] && err 1 "${N1_COLOR}${CBSD_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
	if [ -d "${extra_profile_dir}" ]; then
		if [ -r ${extra_profile_dir}/${template_profile} ]; then
			users_template_profile="${etcdir}/${template_profile} ${extra_profile_dir}/${template_profile}"
		else
			err 1 "${N1_COLOR}${CBSD_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
		fi
	else
		${ECHO} "${W1_COLOR}${CBSD_APP} warning: ${N1_COLOR}no such extra_profile_dir: ${N2_COLOR}${extra_profile_dir}${N0_COLOR}" >&2
		err 1 "${N1_COLOR}${CBSD_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
	fi
fi

[ -z "${mnt_start}" ] && mnt_start="0"

[ -n "${ointer}" ] && inter="${ointer}"
[ "${inter}" = "0" ] && export NOINTER=1

if [ "${mnt_start}" != "0" ]; then
	if [ ! -r "${mnt_start}" -o ! -x "${mnt_start}" ]; then
		err 1 "mnt_start script not exist or not executable: ${mnt_start}"
	fi
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Execute mnt_start script: ${N2_COLOR}${mnt_start}${N0_COLOR}..."
	# external mount, reset zfsfeat
	zfsfeat=0
	[ ! -d "${data}" ] && ${MKDIR_CMD} -m 0770 ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${mnt_start} -d ${data} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}
	_ret=$?
	if [ ${_ret} -ne 0 ]; then
		err 1 "${W1_COLOR}error: ${N1_COLOR}mnt_start script failed: ${N2_COLOR}${mnt_start} -d ${data} -f ${jailfstabdir} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}${N0_COLOR}"
	fi
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${data} ${jailsysdir}/${jname}

	create_fs ${data}
	[ $? -ne 0 ] && err 1 "${N1_COLOR}create_fs failed${N0_COLOR}"
	vm_zfs_guid="0"
fi

if [ "${zfsfeat}" = "1" -a "${mnt_start}" = "0" ]; then
	readconf zfs.conf
	. ${subrdir}/zfs.subr
	DATA=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} )
	_msg=$( ${ZFS_CMD} create -o mountpoint=${workdir}/vm/${jname} ${DATA}/${jname} )
	_res=$?
	if [ ${_res} -ne 0 ]; then
		echo "${_msg}"
		exit ${_res}
	fi

	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${workdir}/vm/${jname}
	${CHMOD_CMD} 0770 ${workdir}/vm/${jname}

	vm_zfs_guid=$( ${ZFS_CMD} get -Ho value guid ${DATA}/${jname} 2>/dev/null )
	[ -z "${vm_zfs_guid}" ] && vm_zfs_guid="0"

	[ -z "${data}" ] && data="${jaildatadir}/${jname}-${jaildatapref}"
	olddata="${data}"
	data="${workdir}/vm/${jname}"
	/usr/local/cbsd/misc/cbsdsysrc -qf ${temprcconf} data="${data}" > /dev/null 2>&1
	${LN_CMD} -sf ${data} ${olddata}
	${LN_CMD} -sf ${data} ${jailsysdir}/${jname}
else
	# no mnt_start and no ZFS
	[ ! -d "${data}" ] && ${MKDIR_CMD} -m 0770 ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${data} ${jailsysdir}/${jname}
	vm_zfs_guid="0"
fi

. ${subrdir}/build.subr
. ${temprcconf}

case "${platform}" in
	Linux)
		conf_owner=$( ${STAT_CMD} -c "%u" ${jconf_tmp} )
		conf_group=$( ${STAT_CMD} -c "%g" ${jconf_tmp} )
		;;
	*)
		conf_owner=$( ${STAT_CMD} -f "%u" ${jconf_tmp} )
		conf_group=$( ${STAT_CMD} -f "%g" ${jconf_tmp} )
		;;
esac

${TRUNCATE_CMD} -s0 ${jconf_tmp}

# Merge with default and profile settings ( users_template_profile can be '${etcdir}/${template_profile}' + '${extra_profile_dir}/${template_profile}"
for merge_me in ${etcdir}/defaults/qemu-default-default.conf ${etcdir}/defaults/${template_profile} ${etcdir}/qemu-default-default.conf ${etcdir}/${template_profile} ${temprcconf}; do
	[ ! -r "${merge_me}" ] && continue;
	tmp_merge=$( ${MKTEMP_CMD} )
	merge from=${jconf_tmp} to=${merge_me} out=${tmp_merge}
	[ -f ${tmp_merge} ] && ${MV_CMD} ${tmp_merge} ${jconf_tmp}
done

# make permission for group write
${CHOWN_CMD} ${conf_owner}:${conf_group} ${jconf_tmp}

. ${jconf_tmp}

# minimal config allows not to specify rcconf variable
# e.g: minimal config
# jname="freebsd1"
# imgsize="10g";
# vm_os_profile="FreeBSD-x64-12.0";
# vm_profile="FreeBSD-x64-12.0"
[ -z "${rcconf}" ] && rcconf="${jailrcconfdir}/rc.conf_${jname}"

[ -n "${oremovejconf}" ] && removejconf="${oremovejconf}"

if [ ${removejconf} = "1" ]; then
	trap "${RM_CMD} -f ${temprcconf} ${jconf} ${jconf_tmp}" HUP INT ABRT BUS TERM  EXIT
else
	trap "${RM_CMD} -f ${temprcconf} ${jconf_tmp}" HUP INT ABRT BUS TERM  EXIT
fi

[ -z "${jname}" ] && log_err 1 "${N1_COLOR}No such jname variable${N0_COLOR}"

if [ "${vm_os_type}" = "freebsd" -a "${from_jail}" = "1" ]; then
	# change emulator type for jcreate
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} emulator="jail" > /dev/null 2>&1

	if [ -n "${jprofile}" ]; then
		. ${subrdir}/settings-tui.subr
		if [ -r "${etcdir}/jail-freebsd-${jprofile}.conf" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Use profile: ${N2_COLOR}${etcdir}/jail-freebsd-${jprofile}.conf${N0_COLOR}"
			merge_apply_profiles ${etcdir}/jail-freebsd-${jprofile}.conf ${jconf_tmp}
		elif [ -r "${etcdir}/defaults/jail-freebsd-${jprofile}.conf" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Use profile: ${N2_COLOR}${etcdir}/defaults/jail-freebsd-${jprofile}.conf${N0_COLOR}"
			merge_apply_profiles ${etcdir}/defaults/jail-freebsd-${jprofile}.conf ${jconf_tmp}
		fi
	fi

	/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} emulator="qemu" > /dev/null 2>&1
fi

# adjust some missed variabled
readconf vnc.conf
readconf spice.conf
[ -z "${qemu_vnc_resolution}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} qemu_vnc_resolution="${default_vnc_width}x${default_vnc_height}"  > /dev/null 2>&1
[ -z "${qemu_vnc_tcp_bind}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} qemu_vnc_tcp_bind="${default_vnc_tcp_bind}"  > /dev/null 2>&1
[ -z "${qemu_vnc_resolution}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} qemu_spice_resolution="${default_spicy_width}x${default_spicy_height}"  > /dev/null 2>&1
[ -z "${qemu_vnc_tcp_bind}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} qemu_spice_tcp_bind="${default_spicy_tcp_bind}"  > /dev/null 2>&1

if [ -z "${cd_vnc_wait}" ]; then
	case "${default_vnc_wait}" in
		auto|1)
			/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} cd_vnc_wait="1"  > /dev/null 2>&1
			;;
		*)
			/usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} cd_vnc_wait="0"  > /dev/null 2>&1
			;;
	esac
fi

[ -z "${vnc_password}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} vnc_password="${default_vnc_password}" > /dev/null 2>&1
[ -z "${spicy_password}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} spicy_password="${default_spicy_password}" > /dev/null 2>&1

# re-read jail params and apply personal after profile
. ${jconf_tmp}

# extra check
# args deps:
#  - interface2 required for ci_interface2
#  - ci_interface2 required for ci_ip4_addr2
# when ci_interface2 not empty - inherit interface2 settings
[ -n "${oci_interface2}" ] && ci_interface2="${ci_interface2}"
[ -n "${ointerface2}" ] && interface2="${ointerface2}"
[ -n "${ci_ip4_addr2}" -a -z "${ci_interface2}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: ci_ip4_addr2 set, but ci_interface2 is empty${N0_COLOR}"
[ -z "${interface2}" -a -n "${ci_interface2}" ] && interface2="${ci_interface2}"

# apply pkglist from tpl_pkglist
[ -n "${tpl_pkglist}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jconf_tmp} pkglist="${tpl_pkglist}" > /dev/null 2>&1

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_qemu_queue_name}" ] && ${cbsd_queue_backend} cbsd_qemu_queue_name=${cbsd_qemu_queue_name} id=${jname} cmd=qcreate vm_ram=${vm_ram} vm_cpus=${vm_cpus} vm_os_type=${vm_os_type} astart=${astart} protected=${protected} vnc_port='0' status=1
fi

# ip validate
if [ -n "${interface}" -a "${inteface}" != "0" ]; then
	### CHECK FOR IP ( 1 - check for interfaces) ####
	checkip ip=${ips} check=1 > /dev/null 2>&1
	case $? in
		0)
			log_err 1 "${N1_COLOR}Ip not in pool range${N0_COLOR}"
			;;
		1)	;;
		2)
			${ECHO} "${N1_COLOR}Warning:${N2_COLOR} Ip already exists in LAN${N0_COLOR}"
			;;
		*)
			log_err 1 "Unknown code from checkip"
			;;
	esac
fi

# imgsize validate
if [ -n "${imgsize_min}" -a -n "${imgsize}" ]; then

	if ! is_number "${imgsize_min}"; then
		# imgsize_min is number. assume value in bytes
		imgsize_min_bytes="${imgsize_min}"
	else
		imgsize_min_bytes=$( get_bytes ${imgsize_min} )
	fi

	[ -z "${imgsize_min_bytes}" -o "${imgsize_min_bytes}" = "0" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to convert imgsize to bytes: ${N2_COLOR}${imgsize_min}${N0_COLOR}"

	imgsize_min_human_mb=$(( imgsize_min_bytes / 1024 / 1024 ))

	if ! is_number "${imgsize}"; then
		# imgsize is number. assume value in bytes
		imgsize_bytes="${imgsize}"
	else
		imgsize_bytes=$( get_bytes ${imgsize} )
	fi

	imgsize_human_mb=$(( imgsize_bytes / 1024 / 1024 ))

	if [ ${imgsize_bytes} -lt ${imgsize_min_bytes} ]; then
		# not fatal?
#		if [ "${zfsfeat}" = "1" ]; then
#			${ZFS_CMD} destroy ${DATA}/${jname}
#		fi
#		err 1 "${N1_COLOR}imgsize too small: ${N2_COLOR}${imgsize_bytes} ( ${imgsize_human_mb}mb ) < ${imgsize_min_bytes} ( ${imgsize_min_human_mb}mb )${N0_COLOR}"
		${ECHO} "${W1_COLOR}imgsize too small: ${N2_COLOR}${imgsize_bytes} ( ${imgsize_human_mb}mb ) < ${imgsize_min_bytes} ( ${imgsize_min_human_mb}mb )${N0_COLOR}"
	fi
fi

# vm_ram validate
if [ -n "${vm_ram_min}" -a -n "${vm_ram}" ]; then

	if ! is_number "${vm_ram_min}"; then
		# vm_ram_min is number. assume value in bytes
		vm_ram_min_bytes="${vm_ram_min}"
	else
		vm_ram_min_bytes=$( get_bytes ${vm_ram_min} )
	fi

	vm_ram_min_human_mb=$(( vm_ram_min_bytes / 1024 / 1024 ))

	if ! is_number "${vm_ram}"; then
		# vm_ram is number. assume value in bytes
		vm_ram_bytes="${vm_ram}"
	else
		vm_ram_bytes=$( get_bytes ${vm_ram} )
	fi

	vm_ram_human_mb=$(( vm_ram_bytes / 1024 / 1024 ))

	if [ ${vm_ram_bytes} -lt ${vm_ram_min_bytes} ]; then
		if [ "${zfsfeat}" = "1" ]; then
			${ZFS_CMD} destroy ${DATA}/${jname}
		fi
		err 1 "${N1_COLOR}vm_ram too small: ${N2_COLOR}${vm_ram_bytes} ( ${vm_ram_human_mb}mb ) < ${vm_ram_min_bytes} ( ${vm_ram_min_human_mb}mb )${N0_COLOR}"
	fi
fi

defdsk="dsk1.vhd"
defnic="nic1.vhd"

if [ -z "${zfs_snapsrc}" ]; then
	really_create_base
	really_create_vm
	${CP_CMD} ${jconf_tmp} ${rcconf}
else
	if [ ! -d ${data} ]; then
		create_fs ${data}
		[ $? -ne 0 ] && err 1 "${N1_COLOR}create_fs failed${N0_COLOR}"
	fi
fi

[ ! -d "${data}" ] && log_err 1 "Can't create datadir ${data}"
[ ! -d ${jailfstabdir}  ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}

/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-qemudsk.schema qemudsk
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-qemunic.schema qemunic
#/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-qemu-p9shares.schema p9shares
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-qemu-settings.schema settings
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-qemu-pcibus.schema pcibus
/usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-qemu-pcibus.schema pcibus_run

[ -n "${fstablocal}" -a -f "${fstablocal}" ] && ${CP_CMD} ${fstablocal} ${jailfstabdir}/${jailfstabpref}${jname}.local

${CP_CMD} ${jconf_tmp} ${rcconf}
. ${rcconf}

# Finnaly export to SQLite
jregister jname=${jname} mode=new

_res=$?

if [ ${_res} -ne 0 ]; then
	if [ ${quiet} -ne 1 ]; then
		${ECHO}
		${ECHO} "${N1_COLOR}Creating ${jname} failed: ${N2_COLOR}cbsd jregister${N0_COLOR}"
		${ECHO} "${N1_COLOR}Please review bad config file: ${N2_COLOR}/tmp/rc.conf_${jname}${N0_COLOR}"
	fi
	${MV_CMD} ${rcconf} /tmp
	#cleanup
	[ -f "${mount_fstab}" ] && ${RM_CMD} -f ${mount_fstab}
	remove_data_dir ${data}
	exit 1
fi

# copy BHYVE_UEFI_VARS.fd
if [ -n "${vars_img}" -a -r "${srcdir}/iso/${vars_img}" ]; then
	${CP_CMD} "${srcdir}/iso/${vars_img}" ${jailsysdir}/${jname}/BHYVE_UEFI_VARS.fd
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Custom UEFI VARS file: ${N2_COLOR}${srcdir}/iso/${vars_img}${N0_COLOR}"
elif [ -r ${distdir}/upgrade/patch/BHYVE_UEFI_VARS.fd ]; then
	${CP_CMD} ${distdir}/upgrade/patch/BHYVE_UEFI_VARS.fd ${jailsysdir}/${jname}/BHYVE_UEFI_VARS.fd
fi

echo
if [ "${vm_zfs_guid}" != "0" ]; then
	[ -z "${NOINTER}" -o ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Global VM ZFS guid: ${N2_COLOR}${vm_zfs_guid}${N0_COLOR}"
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET vm_zfs_guid=\"${vm_zfs_guid}\"
fi

if [ "${uuid}" = "0" -o -z "${uuid}" ]; then
	uuid=$( ${UUIDGEN_CMD} )
	#[ -z "${NOINTER}" -o ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Global VM UUID: ${N2_COLOR}${uuid}${N0_COLOR}"
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET uuid=\"${uuid}\"
	/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} uuid="${uuid}" > /dev/null
fi

if [ ${quiet} -ne 1 -o -z "${NOINTER}" ]; then
	${ECHO} "${N1_COLOR}To edit VM properties use: ${N2_COLOR}cbsd qconfig jname=${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To start VM use: ${N2_COLOR}cbsd qstart ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To stop VM use: ${N2_COLOR}cbsd qstop ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To remove VM use: ${N2_COLOR}cbsd qdestroy ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}For attach VM console use: ${N2_COLOR}cbsd qlogin ${jname}${N0_COLOR}"
	echo
	${ECHO} "${N1_COLOR}Creating ${jname} complete: ${N2_COLOR}Enjoy!${N0_COLOR}"
fi

${RM_CMD} -f ${rcconf}

if [ -z "${sectorsize}" ]; then
	sectorsize="${default_sectorsize}"
	[ -z "${sectorsize}" ] && sectorsize="4096"
fi

if [ ${zfsfeat} -eq 1 -a "${imgtype}" = "zvol" ]; then
	. ${subrdir}/zfs.subr
	dsk_zfs_guid=$( get_dsk_zfs_guid -p ${data}/${defdsk} 2>&1 )
	_ret=$?
	[ ${_ret} -ne 0 ] && dsk_zfs_guid="0"
fi

[ -z "${dsk_zfs_guid}" ] && dsk_zfs_guid="0"

dsk_bsize=0

if is_number "${imgsize}"; then
	if conv2bytes ${imgsize}; then
		dsk_bsize="${convval}"
	else
		dsk_bsize=0
	fi
else
	# already on bytes ?
	dsk_bsize="${imgsize}"
fi

if [ -n "${qemu_virtio_type}" ]; then
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO qemudsk ( jname,dsk_controller,dsk_path,dsk_slot,dsk_size,dsk_zfs_guid ) VALUES ( \"${jname}\",\"${qemu_virtio_type}\",\"${defdsk}\","0",\"${dsk_bsize}\",\"${dsk_zfs_guid}\" )"
else
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO qemudsk ( jname,dsk_path,dsk_slot,dsk_size,dsk_zfs_guid ) VALUES ( \"${jname}\",\"${defdsk}\","0",\"${dsk_bsize}\",\"${dsk_zfs_guid}\" )"
fi

if [ -n "${nic_driver}" ]; then
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO qemunic ( jname,nic_driver,nic_parent ) VALUES ( \"${jname}\", \"${nic_driver}\", \"${interface}\" )"
else
	${miscdir}/sqlcli ${jailsysdir}/${jname}/local.sqlite "INSERT INTO qemunic ( jname,nic_parent ) VALUES ( \"${jname}\", \"${interface}\" )"
fi

# update state_time
cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET state_time="(strftime('%s','now'))"
cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET created="(strftime('%s','now'))"

# Check if SIZE if valid: can't be smaller then template
if is_number ${vm_ram}; then
	# not number, try to convert
	if conv2bytes "${vm_ram}"; then
		vm_ram="${convval}"
	else
		log_err 1 "${vm_ram} is not number and we can't convert int via conv2bytes"
	fi
fi

if [ -n "${register_iso_as}" -a -n "${register_iso_name}" ]; then
	cd_name="${register_iso_as}"
	cd_path="${srcdir}/iso/${register_iso_name}"

	cd_rec_num=$( cbsdsqlro storage_media SELECT COUNT\(path\) FROM media WHERE name=\"${cd_name}\" AND path=\"${cd_path}\" AND type=\"iso\" AND jname=\"-\" )

	if [ "${cd_rec_num}" = "0" ]; then
		# register new ISO with assignment to this VM
		_res=$( media mode=register name="${register_iso_as}" path="${srcdir}/iso/${register_iso_name}" type=iso jname=${jname} 2>&1 )
		_ret=$?
		[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}bcreate: media register error: ${_res}${N0_COLOR}"
	else
		# we have free/unassignent CD. link to this VM
		cbsdsqlrw storage_media "UPDATE media SET jname=\"${jname}\" WHERE jname=\"-\" AND type=\"iso\" AND name=\"${cd_name}\" AND path=\"${cd_path}\""
	fi
fi

if [ ! -d ${jailsysdir}/${jname} ]; then
	${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jailsysdir}/${jname}
fi

if [ -d "${jailsysskeldir}" ]; then
	# we have custom skeldir. copy
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Applying custom skel system dir template from: ${N2_COLOR}${jailsysskeldir}${N0_COLOR}"
	${RSYNC_CMD} -a ${jailsysskeldir}/ ${jailsysdir}/${jname}/
fi

[ ! -d ${jailsysdir}/${jname}/etc ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}/etc

system_dir="create.d \
facts.d \
master_create.d \
master_poststart.d \
master_poststop.d \
master_prestart.d \
master_prestop.d \
master_reboot.d \
remove.d \
start.d \
stop.d"

for i in ${system_dir}; do
	if [ -n "${systemskeldir}" -a "${systemskeldir}/${i}" ]; then
		[ ! -d ${jailsysdir}/${jname}/${i} ] && ${MKDIR_CMD} -m 0775 -p ${jailsysdir}/${jname}/${i}
		${RSYNC_CMD} -az ${systemskeldir}/${i}/ ${jailsysdir}/${jname}/${i}/
	else
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}qcreate: warning: no such dir: ${N2_COLOR}${systemskeldir}/${i}${N0_COLOR}"
			continue
	fi
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jailsysdir}/${jname}/${i}
done

# is cloud-init-based ?
if [ ${is_cloud} -eq 1 ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}auto-generate cloud-init settings: ${N2_COLOR}${jailsysdir}/${jname}/cloud-init${N0_COLOR}" 1>&2
	# auto adjust some missed settings
	if [ -z "${ci_interface}" ]; then
		ci_interface="${interface}"
		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_interface="${ci_interface}" > /dev/null 2>&1
	fi
	[ -z "${ci_jname}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_jname="${jname}" > /dev/null 2>&1
	[ -z "${ci_fqdn}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_fqdn="${host_hostname}" > /dev/null 2>&1

	if [ -z "${ci_ip4_addr}" ]; then
		${ECHO} "${W1_COLOR}warning: ${N1_COLOR}cloud-init based profile but ci_ip4_addr not set. Force to: ${N2_COLOR}DHCP${N0_COLOR}" 2>&1
		ci_ip4_addr="DHCP"
	fi
	if [ -z "${ci_gw4}" ]; then
		case "${ci_ip4_addr}" in
			[Rr][Ee][Aa][Ll][Dd][Hh][Cc][Pp])
				true
				;;
			*)
				${ECHO} "${W1_COLOR}warning: ${N1_COLOR}cloud-init based profile but ci_gw4 not set. VM without gateway?${N0_COLOR}" 2>&1
				;;
		esac
	fi

	OIFS="${IFS}"
	IFS=","

	normalize_ip4_addr=
	normalize_ci_ip4_addr=

	# normalize IP addresses
	for _pureip in ${ci_ip4_addr}; do
		IFS="${OIFS}"

		case "${_pureip}" in
			[Dd][Hh][Cc][Pp])
				_pureip=$( dhcpd 2>/dev/null )
				if [ $? -eq 2 ]; then
					cbsdlogger NOTICE ${CBSD_APP}: no free IP address for DHCP in nodeippool
					err 1 "${N1_COLOR}${CBSD_APP}: no free IP address for DHCP in nodeippool${N0_COLOR}"
				fi

				_pureip="${_pureip}/24"		# todo: hardcoded mask
				_mod=1
				;;
			[Dd][Hh][Cc][Pp][vV]6)
				_pureip=$( dhcpdv6 2>/dev/null )
				if [ $? -eq 2 ]; then
					cbsdlogger NOTICE ${CBSD_APP}: no free IP address for DHCPv6 in nodeip6pool
					err 1 "${N1_COLOR}${CBSD_APP}: no free IP address for DHCPv6 in nodeip6pool${N0_COLOR}"
				fi

				_pureip="${_pureip}/64"		# todo: hardcoded mask
				_mod=1
				;;
		esac

		iptype ${_pureip}
		_ret=$?

		if [ -z "${normalize_ip4_addr}" ]; then
			normalize_ip4_addr="${IWM}"
		else
			normalize_ip4_addr="${normalize_ip4_addr},${IWM}"
		fi
		if [ -z "${normalize_ci_ip4_addr}" ]; then
			normalize_ci_ip4_addr="${IWM}/24"
		else
			normalize_ci_ip4_addr="${normalize_ci_ip4_addr},${IWM}/24"
		fi

		IFS=","
	done

	IFS="${OIFS}"

	ci_ip4_addr="${normalize_ci_ip4_addr}"
	[ -n "${ci_gw42}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_gw42="${ci_gw42}" > /dev/null 2>&1

	if [ -n "${ci_ip4_addr2}" ]; then
		#todo: normalize
		/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_ip4_addr2="${ci_ip4_addr2}" > /dev/null 2>&1
		normalize_ip4_addr="${normalize_ip4_addr},${ci_ip4_addr2}"
	fi

	/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_ip4_addr="${normalize_ci_ip4_addr}" > /dev/null 2>&1
	qset jname="${jname}" ip4_addr="${normalize_ip4_addr}" > /dev/null 2>&1

	readconf cloud-init.conf

	[ -z "${ci_nameserver_address}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_nameserver_address="${default_ci_nameserver_address}" > /dev/null 2>&1
	[ -z "${ci_nameserver_search}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_nameserver_search="${default_ci_nameserver_search}" > /dev/null 2>&1
	[ -z "${ci_interface_name}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_interface_name="${default_ci_interface_name}" > /dev/null 2>&1
	[ -z "${ci_user_pw_user}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pw_user="${ci_user_pw_user}" > /dev/null 2>&1
	[ -z "${ci_user_pw_root}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pw_root="${ci_user_pw_root}" > /dev/null 2>&1
	[ -z "${ci_user_pw_root}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pw_root="${ci_user_pw_root}" > /dev/null 2>&1
	[ -z "${ci_user_pubkey}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_user_pubkey="${ci_user_pubkey}" > /dev/null 2>&1
	[ -z "${ci_interface2}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_interface2="${ci_interface2}" > /dev/null 2>&1

	# GET MAC
	nic_hwaddr0=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT nic_hwaddr FROM qemunic LIMIT 1 )

	if [ "${nic_hwaddr0}" = "0" ]; then
		# gen MAC
		nic_hwaddr0=$( mac_gen 00:a0:98 )
		cbsdlogger NOTICE ${CBSD_APP}: bcreate for ${jname}: MAC address randomized and updated for nic id 1: ${nic_hwaddr0}
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE qemunic SET nic_hwaddr=\"${nic_hwaddr0}\" WHERE id=\"1\""
	fi

	[ -z "${ci_nic_hwaddr0}" ] && /usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ci_nic_hwaddr0="${nic_hwaddr0}" > /dev/null 2>&1

	#/usr/local/cbsd/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ip4_addr="${ip4_addr}" > /dev/null 2>&1

	case "${vm_os_type}" in
		windows)
			cloudengine="cloudinit-base"
			;;
		*)
			cloudengine="cloud-init"
			;;
	esac

	cbsdlogger NOTICE ${CBSD_APP}: cloudinit mode=gen fromfile=${jailsysdir}/${jname}/rc.conf_${jname} jname=${jname} cloudengine="${cloudengine}"
	cloudinit mode=gen fromfile=${jailsysdir}/${jname}/rc.conf_${jname} jname=${jname} cloudengine="${cloudengine}"
fi

# mark the first disk bootable
cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE qemudsk SET bootable='true' WHERE dsk_path=\"${defdsk}\""

# unset vnc_password when vnc_password=0 (reserved)
[ "${vnc_password}" = "0" ] && cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE settings SET vnc_password=''"

# set default nice from rctl_nice
[ -n "${rctl_nice}" ] && nice="${rctl_nice}"

# rctl
. ${distsharedir}/rctl.conf
for i in ${RCTL} ${RCTL_EXTRA}; do
	_val=
	eval _val="\$rctl_${i}"
	if [ -n "${_val}" ]; then
		jrctl jname=${jname} mode=set ${i}=${_val} > /dev/null 2>&1 || /usr/bin/true
	fi
done

# store profile in jail system dir
storeconf vm-${vm_os_type}-${vm_os_profile}.conf ${jailsysdir}/${jname}/etc/vm-${vm_os_type}-${vm_os_profile}.conf

# create ascii rc.conf file for overwriting values
${CAT_CMD} > ${jailsysdir}/${jname}/etc/rc.conf <<EOF
# The parameters in this file can overwrite the settings from SQLite3 or global params
# e.g:
# qemu_flags="-p 1:1 -p 2:2"
EOF

geniplist		${ip4_addr}		# for ipvX_first_public-like vars
. ${subrdir}/jcreate.subr
export_qemu_data_for_external_hook
external_exec_master_script "master_create.d"

# API compat: ~cbsd/jails-system/info*.*
/usr/local/cbsd/misc/daemonize /usr/local/cbsd/tools/save-jail-info jname=${jname}

jcleanup jname=${jname}

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_qemu_queue_name}" ] && ${cbsd_queue_backend} cbsd_qemu_queue_name=${cbsd_qemu_queue_name} id="${jname}" cmd=qcreate status=2 data_status=0
fi

end_time=$( ${DATE_CMD} +%s )
cbsdlogger NOTICE ${CBSD_APP}: vm ${jname} has been created in $(( end_time - st_time ))s

[ "${runasap}" = "1" ] && qstart inter=0 quiet=${quiet} jname=${jname}

exit 0
