#!/usr/local/bin/cbsd
#v12.1.8
MYARG=""
MYOPTARG="delay jname inter quiet"
MYDESC="Start jail"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

 Start the jail container. When used in a directory with a CBSDfile,
the jstart command only processes the environments described in the CBSDfile.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}delay=${N0_COLOR} - <sec>, delay N secbefore start, mainly to smooth the astart,
          default is: '0', no delay.
 ${N2_COLOR}inter=${N0_COLOR} - set 1 to prevent any questions and to accept answers by default.
 ${N2_COLOR}jname=${N0_COLOR} - target jail. If jail='*' or jail='pri*' then start all jails or
          jails whose names begin with 'pri', e.g. 'prison1', 'prisonX'...
 ${N2_COLOR}quiet=${N0_COLOR} - 0,1: be quiet, dont output verbose message.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd jstart
 # cbsd jstart jname='memcach*'

${H3_COLOR}See also${N0_COLOR}:

 cbsd jstop --help

"

CBSDMODULE="jail"
EXTHELP="wf_jstop_jstart"

. ${subrdir}/nc.subr
. ${subrdir}/tools.subr		# for select_jail_by_list

readconf buildworld.conf
readconf jail-freebsd-default.conf

jname=

if [ -n "${ci_gw4}" -a "${ci_gw4}" != "0" ]; then
	oci_gw4="${ci_gw4}"
else
	if [ -n "${default_ci_gw4}" -a "${default_ci_gw4}" != "0" ]; then
		oci_gw4="${default_ci_gw4}"
	fi
fi

# check for cloud function when CBSDfile exist
Makefile="${CBSD_PWD}/CBSDfile"
if [ ! -r "${Makefile}" ]; then
		[ -z "${1}" ] && select_jail_by_list -s "List of offline jail" -a "Off" -r ${sqlreplica}
fi

delay=0
odelay=
. ${cbsdinit}
ojname="${jname}"
[ -n "${delay}" ] && odelay="${delay}"

. ${subrdir}/cbsdinit.subr
. ${subrdir}/system.subr
. ${subrdir}/universe.subr
. ${subrdir}/ipfw.subr	# fwcounter

[ -z "${quiet}" ] && quiet=0

# adjust jail_list by CBSDfile
if [ -r "${Makefile}" ]; then
	[ -z "${CBSDFILE_RECURSIVE}" ] && ${ECHO} "${N1_COLOR}found CBSDfile: ${N2_COLOR}${Makefile}${N0_COLOR}" 1>&2
	. ${Makefile}
	all_jail_list=$( ${GREP_CMD} -E '^jail_[a-zA-Z0-9_@%:][-a-zA-Z0-9_@%:]*\(\)$' ${Makefile} | ${XARGS_CMD} | ${TR_CMD} -d "()" | ${SED_CMD} s#jail_##g )

	# cbsd jstart <env1> <env2> route
	jname_in_args=0

	if [ -n "${ojname}" ]; then
		jail_list="${ojname}"
	else
		# cbsd jstart <env1> <env2> route
		for i in $*; do
			strpos --str="${i}" --search="="
			if [ $? -eq 0 ]; then
				jname_in_args=1
				# not param=value - jail?
				jname_in_cbsdfile=0
				for j in ${all_jail_list}; do
					[ "${i}" != "${j}" ] && continue
					jname_in_cbsdfile=1
				done
				if [ ${jname_in_cbsdfile} -eq 1 ]; then
					if [ -z "${ojname}" ]; then
						ojname="${i}"
					else
						ojname="${ojname} ${i}"
					fi
				else
					${ECHO} "${N1_COLOR}${CBSD_APP}: env absent in CBSDfile: ${N2_COLOR}${i}${N0_COLOR}" 1>&2
				fi
			fi
		done
		if [ ${jname_in_args} -eq 1 ]; then
			jail_list="${ojname}"
		else
			jail_list="${all_jail_list}"
		fi
	fi

	unset ojname jname

	[ -z "${jail_list}" ] && err 1 "${N1_COLOR}${CBSD_APP}: give me jname${N0_COLOR}"

	# multiple?
	strpos --str="${jail_list}" --search=" "
	if [ $? -eq 0 ]; then
		# alone
		jname="${jail_list}"
		unset jail_list
	fi

	if [ -n "${CLOUD_URL}" -a -n "${CLOUD_KEY}" ]; then
		cbsd_api=1
	else
		cbsd_api=0
	fi
else
	cbsd_api=0
	# trim args from "$*"

	if [ -n "${ojname}" ]; then
		jail_list="${ojname}"
	else
		for i in $*; do
			strpos --str="${i}" --search="="
			[ $? -ne 0 ] && continue

			if [ -z "${jail_list}" ]; then
				jail_list="${i}"
			else
				jail_list="${jail_list} ${i}"
			fi
		done
	fi

	# multiple?
	strpos --str="${jail_list}" --search=" "
	if [ $? -eq 0 ]; then
		# alone
		jname="${jail_list}"
		unset jail_list
	fi
fi

[ -z "${jname}" -a -z "${jail_list}" ] && err 1 "${N1_COLOR}No jail specified${N0_COLOR}"

# for external_exec-related command
. ${subrdir}/jcreate.subr

. ${subrdir}/jstart.subr
. ${subrdir}/time.subr

# MAIN for multiple jails
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

TRAP=""
emulator="jail"	# for jname_is_multiple
# jail_list can be init in CBSDfile case
[ -z "${jail_list}" ] && jname_is_multiple

if [ -n "${jail_list}" ]; then
	TMP_JLIST="${jail_list}"
else
	TMP_JLIST="${jname}"
fi

JLIST=

# check for actual vm list in arg list
jail_num=0
jname=
for i in ${TMP_JLIST}; do
	exist=$( cbsdsqlro local SELECT jname FROM jails WHERE jname=\"${i}\" AND emulator=\"${emulator}\" LIMIT 1 )
	if [ -n "${exist}" ]; then
		if [ -n "${JLIST}" ]; then
			JLIST="${JLIST} ${i}"
		else
			JLIST="${i}"
		fi
		if [ ${jail_num} -eq 0 ]; then
			jname="${i}"
		else
			# when > 1, jname unused below
			unset jname
		fi
		jail_num=$(( jail_num + 1 ))
	fi
done

# this is multiple list, split it by parallel jstart execution
if [ ${jail_num} -gt 1 ]; then
	cbsdlogger NOTICE ${CBSD_APP}: executing for multiple starting: ${JLIST}
	# multiple astart always non interactive
	export inter=0

	st_time=$( ${DATE_CMD} +%s )

	if [ -z "${parallel}" -o "${parallel}" = "0" ]; then
		for jname in ${JLIST}; do
			/usr/local/bin/cbsd jstart inter=0 jname=${jname}
		done
	else
		_seq=0
		wait_pid_files=

		for jname in ${JLIST}; do
			_boot_delay=$( cbsdsqlro local SELECT boot_delay FROM jails WHERE jname=\"${jname}\" LIMIT 1 )

			if [ -z "${_boot_delay}" ]; then
				_boot_delay=0
			fi

			if [ ${_boot_delay} -eq 0 ]; then
				_boot_delay="${_seq}"
			fi

			env NOINTER=1 /usr/local/cbsd/misc/daemonize -E NOINTER=1 -p ${ftmpdir}/jstart.${jname}.$$ /usr/local/bin/cbsd jstart inter=0 jname=${jname} delay=${_boot_delay}
			if [ -z "${wait_pid_files}" ]; then
				wait_pid_files="${ftmpdir}/jstart.${jname}.$$"
			else
				wait_pid_files="${wait_pid_files} ${ftmpdir}/jstart.${jname}.$$"
			fi

			_seq=$(( _seq + 1 ))
		done

		#lets save .pid file
		sleep 1

		# wait loop
		for i in ${wait_pid_files}; do
			[ ! -r ${i} ] && continue
			cbsd_pwait --pid=$( ${CAT_CMD} ${i} ) --timeout=${parallel}
		done

		wait_for_fpid -a start -t ${parallel}
	fi

	end_time=$( ${DATE_CMD} +%s )
	diff_time=$(( end_time - st_time ))
	diff_time=$( displaytime ${diff_time} )
	${ECHO} "${N1_COLOR}${CBSD_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"
	cbsdlogger NOTICE ${CBSD_APP}: executing for multiple done in ${diff_time}: ${JLIST}
	err 0 "${N1_COLOR}Multiple start: ${N2_COLOR}done${N0_COLOR}"
fi

# MAIN
# single jname
[ -z "${jname}" ] && jname="${1}"
[ -z "${jname}" ] && err 1 "${N1_COLOR}Give me jail name${N0_COLOR}"

. ${subrdir}/rcconf.subr

if [ $? -eq 1 ]; then
	# remote start
	[ ${sqlreplica} -eq 0 ] && err 1 "${N1_COLOR}No such jail: ${N2_COLOR}${jname}${N0_COLOR}"
	remotenode=$( jwhereis ${jname} )
	[ -z "${remotenode}" ] && err 1 "${N1_COLOR}No such jail: ${N2_COLOR}${jname}${N0_COLOR}"
	for i in ${remotenode}; do
		if [ "${i}" = "${nodename}" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote jstart: found on nodename ${N2_COLOR}${nodename}${N1_COLOR}. Skipped${N0_COLOR}"
			continue
		fi
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote jstart: ${N2_COLOR}${jname} ${N1_COLOR}on${N2_COLOR} ${i}${N0_COLOR}"
		rexe node=${i} cbsd jstart jname=${jname}
		if [ $? -eq 0 ]; then
			# updating state and put task for retrinv inventory
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Updating inventory...${N0_COLOR}"
			task autoflush=2 mode=new retrinv node=${i} tryoffline=1 data=db > /dev/null 2>&1
		fi
	done
	exit 0
fi

# mainly to smooth mass start in astart.
[ -n "${odelay}" ] && delay="${odelay}"
if [ ${delay} -ne 0 ]; then
	cbsdlogger NOTICE ${CBSD_APP}: delayed boot for ${jname} via delay= params: ${delay} sec.
	sleep ${delay}
fi

# start jail
st_time=$( ${DATE_CMD} +%s )

if [  "${ver}" = "native" ]; then
	tmpver=$( ${UNAME_CMD} -r )
	ver=${tmpver%%-*}
	unset tmpver
	[ "${stable}" = "1" ] && ver=${ver%%.*}
	cbsdsqlrw local "UPDATE jails SET ver=\"${ver}\" WHERE jname=\"${jname}\""
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}adjust version: ${N2_COLOR}native -> ${ver}${N0_COLOR}"
fi

if [ "${arch}" = "native" ]; then
	arch=$( ${UNAME_CMD} -m )
	[ "${arch}" = "x86_64" ] && arch="amd64"

	init_target_arch

	cbsdsqlrw local "UPDATE jails SET arch=\"${arch}\" WHERE jname=\"${jname}\""
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}adjust version: ${N2_COLOR}native -> ${ver}${N0_COLOR}"
fi

over="${ver}"

# Determine stable value. Must be after buildconf
strpos --str="${over}" --search="."

# auto-detect for stable/release
pos=$?
if [ ${pos} -eq 0 ]; then
	stable=1
	ostable=1
else
	stable=0
	ostable=0
fi

[ ${status} -eq 2 ] && err 1 "${N1_COLOR}Jail in slave mode. Please ${N2_COLOR}cbsd jswmode mode=master${N1_COLOR} first${N0_COLOR}"
[ ${status} -eq 3 ] && err 1 "${N1_COLOR}Jail in maintenance mode${N0_COLOR}"
[ ${jid} -ne 0 ] && err 1 "${N1_COLOR}Jail ${jname} already running, jid: ${N2_COLOR}${jid}${N0_COLOR}"

if [ "${vnet}" = "1" -a "${vimage_feature}" = "0" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} have vnet=1 flags but your kernel is not support VIMAGE${N0_COLOR}"
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Please recompile kernel with: ${N2_COLOR}options VIMAGE${N0_COLOR}"
	sleep 3
	vnet=0
fi

[ "${emulator}" = "bhyve" -a -z "${mdsize}" ] && err 1 "${N1_COLOR}Bhyve required for file image${N0_COLOR}"

TRAP=""

#Check for shared lock
jaillock="${jailsysdir}/${jname}/locked"
if [ -f "${jaillock}" ]; then
	masterhost=$( ${CAT_CMD} ${jaillock} )
	if [ "${masterhost}" = "${nodename}" -o -z "${masterhost}" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: remove my stale lock file: ${jaillock}
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remove my stale lock file: ${N2_COLOR}${jaillock}${N0_COLOR}"
		${RM_CMD} -f ${jaillock}
	else
		cur_time=$( ${DATE_CMD} +%s )
		eval $( ${STAT_CMD} -s ${jaillock} )
		difftime=$(( ( cur_time - st_mtime ) / 60 ))

		# 30 minutes outdated lock
		if [ ${difftime} -gt 30 ]; then
			cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file but lock file age is too old: ${difftime}. Removing!
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} locked by ${N2_COLOR}${masterhost}${N1_COLOR} node${N0_COLOR}"
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}But lock age is too old: ${difftime} min. Removing!!!${N0_COLOR}"
			${RM_CMD} -f ${jaillock}
		else
			# still fresh
			cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file and lock file age time is fresh: ${difftime}.
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} locked by ${N2_COLOR}${masterhost}${N1_COLOR} node, lock age: ${difftime} min.${N0_COLOR}"
			log_err 1 "${N1_COLOR}You may remove the lockfile if you believe that jail is not running on this node: ${N2_COLOR}rm -f ${jaillock}${N0_COLOR}"
		fi
	fi
fi

TRAP="${TRAP} ${RM_CMD} -f ${jaillock};"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

#Check for md vnode backend
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	MDFILE="${jailsysdir}/${jname}/image.dat"
	if [ ! -f "${MDFILE}" -a ! -h "${MDFILE}" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: jail: ${jname}: no such ${MDFILE} but mdsize flags is not null, skip
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}No such ${MDFILE} but mdsize flags is not null. Skip${N0_COLOR}" && continue
	fi
fi

# Update Redis
if [ "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
	cbsdredis hset "jail:${jname}" node "${nodename}" status 1 || echo "WARNING: failed to update Redis!"
	cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":1}'
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	if [ -z "${cbsd_queue_backend}" ]; then
		MOD_CBSD_QUEUE_DISABLED="1"
	else
		[ -n "${cbsd_jail_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_jail_queue_name} id=${jname} cmd=jstart status=1 workdir="${workdir}"
	fi
fi

[ "${emulator}" = "bhyve" ] && start_bhyve

init_target_arch
init_basedir

if [ ${baserw} -eq 0 ]; then
	if [ -r "${BASE_DIR_LOCKFILE}" ]; then
		locked_by=$( ${CAT_CMD} ${BASE_DIR_LOCKFILE} )
		log_err 1 "${N1_COLOR}basedir locked: ${N2_COLOR}${BASE_DIR}${N1_COLOR}, by pid: ${N2_COLOR}${locked_by}${N1_COLOR}. Please try later or remove ${BASE_DIR_LOCKFILE}${N0_COLOR}"
	fi
fi

jcleanup jname=${jname}

[ ! -d "${path}" ] && ${MKDIR_CMD} ${path}
[ -z "${mnt_start}" ] && mnt_start="0"

[ -n "${ointer}" ] && inter="${ointer}"
[ -n "${inter}" ] && export NOINTER=1

if [ "${mnt_start}" != "0" ]; then
	if [ ! -r "${mnt_start}" -o ! -x "${mnt_start}" ]; then
		err 1 "mnt_start script not exist or not executable: ${mnt_start}"
	fi
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Execute mnt_start script: ${N2_COLOR}${mnt_start}${N0_COLOR}..."
	# external mount, reset zfsfeat
	zfsfeat=0
	[ ! -d ${data} ] && ${MKDIR_CMD} -p ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -p ${jailsysdir}/${jname}
	${mnt_start} -d ${data} -f ${jailfstabdir}/${jname} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}
	_ret=$?
	if [ ${_ret} -ne 0 ]; then
		err 1 "${W1_COLOR}error: ${N1_COLOR}mnt_start script failed: ${N2_COLOR}${mnt_start} ${jname}${N0_COLOR}"
	fi
fi

check_environment_script "tests"

[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && /usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-jailnic.schema jailnic

# check for DHCP in ip4_addr string and replace them by ip
OIFS="${IFS}"
IFS=","
_tmp_ip4_addr=
_mod=0

for _tmp in ${ip4_addr}; do
	case ${_tmp} in
		[Dd][Hh][Cc][PP])
			_tmp=$( dhcpd )
			[ $? -eq 2 ] && log_err 1 "${N1_COLOR}No free IP address for DHCP in nodeippool${N0_COLOR}"
			cbsdlogger NOTICE ${CBSD_APP}: set ip4_addr for ${jname} via DHCP: ${_tmp}
			_mod=1
			;;
		[Dd][Hh][Cc][PP][vV]6)
			_tmp=$( dhcpdv6 )
			[ $? -eq 2 ] && log_err 1 "${N1_COLOR}No free IP6 address for DHCP in nodeip6pool${N0_COLOR}"
			cbsdlogger NOTICE ${CBSD_APP}: set ip4_addr for ${jname} via DHCPv6: ${_tmp}
			_mod=1
			;;
		*)
			;;
	esac

	if [ -z "${_tmp_ip4_addr}" ]; then
		_tmp_ip4_addr="${_tmp}"
	else
		_tmp_ip4_addr="${_tmp_ip4_addr},${_tmp}"
	fi
done

IFS="${OIFS}"

if [ ${_mod} -eq 1 ]; then
	jset jname=${jname} ip4_addr="${_tmp_ip4_addr}"
	ip4_addr="${_tmp_ip4_addr}"
fi

geniplist ${ip4_addr}

if [ -n "${interface}" -a "${interface}" != "0" -a "${vnet}" -eq "0" ]; then

	if [ "${interface}" != "auto" ]; then
		# check for interface existance
		case "${platform}" in
			Linux)
				${IP_CMD} link show ${interface} >/dev/null 2>&1
				_ret=$?
				;;
			*)
				${IFCONFIG_CMD} ${interface} >/dev/null 2>&1
				_ret=$?
				;;
		esac

		if [ ${_ret} -ne 0 ]; then
			${ECHO} "${W1_COLOR}Warning: ${N1_COLOR}interface not exist for jail ${jname}: ${N2_COLOR}${interface}${N0_COLOR}"
			cbsdlogger NOTICE ${CBSD_APP}: Warning: interface not exist for jail ${jname}: ${interface}
		fi
	fi

	### CHECK FOR IP ####
	for ips in ${IPS}; do
		iptype "${ips}" ||true
		[ -z "${IWM}" ] && continue
		[ -n "${VHID}" ] && continue
		#prevent to use nodeip
		[ "${IWM}" = "${nodeip}" ] && err 1 "${N1_COLOR}Error: Jail can not take nodeip when interface is not equal 0: ${N2_COLOR}${nodeip}${N0_COLOR}"
		checkip ip=${IWM} check=1
		IPUSE=$?
		case ${IPUSE} in
			0)
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ip ${IWM} not in pool range${N0_COLOR}"
				continue
			;;
			2)
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Ip ${IWM} already exists in LAN${N0_COLOR}"
				continue
			;;
		esac
	done
fi

# args for makejconf
epairb_list=
# for traffic count
epaira_list=

# extract and export/rewrite interface variable from jailnic for vnet-based jails
if [ "${vnet}" = "1" ]; then
	# for vnet we can make another action
	. ${subrdir}/vnet.subr

	interfaces=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite "SELECT name FROM jailnic" | while read _int; do
		printf "${_int} "
	done ) || err 1 "${N1_COLOR}jstart: error get interfaces name for vnet nic map${N0_COLOR}"

	eth_seq=0
	printf "${N1_COLOR}create epair: "

	for i in ${interfaces}; do
		nic_hwaddr=
		nic_parent=
		nic_address=
		nic_type=
		_A=
		_A=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite "SELECT nic_parent,nic_address,nic_hwaddr FROM jailnic WHERE name='${i}'" )

		sqllist "${_A}" nic_parent nic_address nic_hwaddr

		tmp_type=${nic_parent%%-*}		# cut all before '#'
		tmp_iface=${nic_parent##*-}		# cut all after '#'

		case "${tmp_type}" in
			ppt|epair|ovs)
				nic_type="${tmp_type}"
				i="${tmp_iface}"
				;;
			*)
				nic_type="epair"		# epair by default
				;;
		esac

		unset tmp_type tmp_type

		# when ip=0 and interface=auto we must use default interface for upstream
		[ "${nic_parent}" = "0" -o "${nic_parent}" = "auto" ] && nic_parent=$( getnics-by-ip ip=0.0.0.0 skip=bridge )

		case "${nic_parent}" in
			vpc-*)
				VPC_ROOT_DIR="${dbdir}/vpc"
				_arg_len=$( strlen ${nic_parent} )
				vpc_name=$( substr --pos=5 --len=${_arg_len} --str="${nic_parent}" )
				_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
				[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

				mybridge=
				if ! mybridge=$( get_my_device vpc vpc-${vpc_name} ); then
					err 1 "${N1_COLOR}${CBSD_APP} failed to get VPC bridge: ${mybridge}${N0_COLOR}"
				fi

				${IFCONFIG_CMD} ${mybridge} > /dev/null 2>&1
				_ret=$?
				if [ ${_ret} -ne 0 ]; then
					err 1 "${N1_COLOR}no such bridge interface: ${N2_COLOR}${mybridge}${N0_COLOR}"
				fi
				;;
			bridge*)
				mybridge="${nic_parent}"
				;;
			cbsdvale*)
				mybridge="${nic_parent}"
				;;
			ppt-*)
				mybridge=
				true
				;;
			*)
				if ! mybridge=$( get_my_device bridge ${nic_parent} ); then
					err 1 "${N1_COLOR}Error: Cant get_my_device for ${nic_parent}: ${mybridge}${N0_COLOR}"
				fi
		esac

		echo "NT: ${nic_type}"

		if [ "${nic_type}" = "ppt" ]; then
			myepair="${i}"
			if [ -z "${epairb_list}" ]; then
				epairb_list="${myepair}"
			else
				epairb_list="${epairb_list},${myepair}b"
			fi
		else
			myepair=$( get_my_epair ${mybridge} )
			[ $? -eq 1 ] && err 1 "${N1_COLOR}Error: Cant get_my_epair by: ${N2_COLOR}${mybridge}${N0_COLOR}"
			if [ -z "${epairb_list}" ]; then
				epaira_list="${myepair}a"
				epairb_list="${myepair}b"
			else
				epaira_list="${epaira_list},${myepair}a"
				epairb_list="${epairb_list},${myepair}b"
			fi
		fi

		case "${nic_parent}" in
			cbsdvale*)
				strpos --str="${nic_parent}" --search="_"
				_pos=$?
				if [ ${_pos} -eq 0 ]; then
					# not vale_XXX form
					err 1 "${N1_COLOR}${CBSD_APP}: vale switch not vale_XXX form: ${N2_COLOR}${nic_parent}${N0_COLOR}"
				else
					_arg_len=$( strlen ${nic_parent} )
					_pref=$(( _arg_len - _pos ))
					_vale_arg=$( substr --pos=0 --len=${_pos} --str="${nic_parent}" )
					_sw=$( substr --pos=$(( ${_pos} +2 )) --len=${_pref} --str="${nic_parent}" )
				fi
				valeid=$( cbsdsqlro local "SELECT idx FROM vale WHERE name='${_sw}'"  )
				[ -z "${valeid}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: can't determine vale nic for: ${nic_parent}, sw: ${_sw}${N0_COLOR}"
				#mytap="vale${valeid}:${jname}"
				cbsdlogger NOTICE "${CBSD_APP}: vale switch id: ${valeid} (vale${valeid})"
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CBSD_APP}: attach to vale switch id: ${valeid} (${N2_COLOR}vale${valeid}${N1_COLOR})${N0_COLOR}"
				${distdir}/tools/vale-ctl -h vale${valeid}:${myepair}a
		esac

		printf "${H3_COLOR}${myepair}(type:${nic_type}):${N2_COLOR}${nic_parent} "

		case "${nic_type}" in
			ppt)
				#${IFCONFIG_CMD} ${myepair} description ${jname}-eth${eth_seq} up
				${IFCONFIG_CMD} ${myepair} description ${jname}-${myepair} up
				;;
			*)
				${IFCONFIG_CMD} ${myepair}a description ${jname}-eth${eth_seq} up
				;;
		esac

		eth_seq=$(( eth_seq + 1 ))

		case "${nic_type}" in
			ppt)
				true
#				TRAP="${TRAP} ${IFCONFIG_CMD} eth${eth_seq} name ${myepair};"
#				trap "${TRAP}" HUP INT ABRT BUS TERM EXIT
				;;
			*)
				TRAP="${TRAP} ${IFCONFIG_CMD} ${myepair}a destroy;"
				trap "${TRAP}" HUP INT ABRT BUS TERM EXIT
				;;
		esac

		# MAC MGMT
		if [ "${nic_hwaddr}" = "0" ]; then
			nic_hwaddr=$( mac_gen 00:a0:98 )
			cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE jailnic SET nic_hwaddr='${nic_hwaddr}' WHERE name='${i}'"
		fi

		_iface_rename=0

		case "${nic_type}" in
			ppt)
				cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: set hwaddr for vnet interface ${myepair}: ${nic_hwaddr}
				${IFCONFIG_CMD} ${myepair} ether ${nic_hwaddr}
				;;
			*)
				cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: set hwaddr for vnet interface ${myepair}b: ${nic_hwaddr}
				_iface_rename=1
				${IFCONFIG_CMD} ${myepair}b ether ${nic_hwaddr}
				if [ "${nic_address}" != "0" ]; then
					${IFCONFIG_CMD} ${myepair}a inet ${nic_address} up
				fi
		esac
	done

	[ ${quiet} -ne 1 ] && ${ECHO} "${N0_COLOR}"
fi	# vnet -eq 1

#test for zfs mounted & mount if not
case ${zfsfeat} in
	1)
		. ${subrdir}/zfs.subr
		zfsmnt ${data}
		if [ $? -eq 2 ]; then
			# ZFS encryption?
			zfs_encryption_val=$( ${ZFS_CMD} get -Ho value encryption ${ZPOOL} 2>/dev/null )
			if [ -n "${zfs_encryption_val}" -a "${zfs_encryption_val}" != "off" ]; then
				${ECHO} "${N1_COLOR}${CBSD_APP}: encrypted dataset: ${N2_COLOR}${zfs_encryption_val}${N0_COLOR}"
				zfs_encryption_keyformat=$( ${ZFS_CMD} get -Ho value keyformat ${ZPOOL} 2>/dev/null )
				${ECHO} "${N1_COLOR}${CBSD_APP}: encryption keyformat: ${N2_COLOR}${zfs_encryption_keyformat}${N0_COLOR}"
				if [ "${zfs_encryption_keyformat}" = "passphrase" ]; then
					if [ -n "${NOINTER}" -o "${inter}" = "1" ]; then
						err 0 "${N1_COLOR}${CBSD_APP} keyformat is passphrase, skipp for non-interactive mode${N0_COLOR}"
					else
						${ZFS_CMD} load-key ${ZPOOL}
						# 255 code valid here, already loaded
					fi
				fi
			fi
			${ZFS_CMD} mount "${ZPOOL}"
			is_mounted ${data} || err 1 "${N1_COLOR}${CBSD_APP}: unable to mount zfs pool: ${N2_COLOR}${ZPOOL} -> ${data}${N0_COLOR}"
		fi
	;;
esac

if [ "${ver}" != "empty" ]; then
	if [ ${baserw} -eq 1 ]; then
		path=${data}
		[ ! -f "${path}/bin/sh" ] && switch_baserw ${path} 1
		[ -f ${mount_fstab} ] && ${RM_CMD} -f ${mount_fstab}
	fi

	if [ ${baserw} -eq 0 -a ! -f "${mount_fstab}" ]; then
		switch_baserw ${path} 2
	fi
fi

# MD area
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	cbsd mountmd mdfile="${MDFILE}" jroot="${data}"
fi

if [ "${ver}" != "empty" ]; then
	[  -d "${data}/etc" -o -d "${data}/bin" ] || err 1 "${N1_COLOR}No such data structure: ${N2_COLOR}${data}${N0_COLOR}"
fi

# cp local default resolv.conf skel
if [ ${floatresolv} -eq 1 ]; then
	[ -d ${data}/etc ] && makeresolv jname=${jname}
fi

MOUNTOPT=""

if [ "${mount_src}" = "1" ]; then
	SRCDIR="${srcdir}/src_${ver}/src"
	MOUNTOPT="${MOUNTOPT} -s ${SRCDIR}"
fi

if [ "${mount_obj}" = "1" ]; then
	SRCDIR="${srcdir}/obj_${arch}_${ver}/obj"
	MOUNTOPT="${MOUNTOPT} -o ${SRCDIR}"
fi

if [ "${mount_kernel}" = "1" ]; then
	if [ -d ${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel ]; then
		SRCDIR="${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel"
		MOUNTOPT="${MOUNTOPT} -k ${SRCDIR}"
	fi
fi

if [ "${mount_ports}" = "1" ]; then
	case "${platform}" in
		DragonFly)
			MOUNTOPT="${MOUNTOPT} -p /usr/dports"
			;;
		*)
			MOUNTOPT="${MOUNTOPT} -p /usr/ports -d ${data}/var/cache/distfiles"
			;;
	esac
fi

[ "${ver}" != "empty" ] && mountbase -v ${ver} -a ${arch} ${MOUNTOPT}

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
with_zfs_attach="${jailsysdir}/${jname}/with_zfs_attach"
[ -r ${with_zfs_attach} ] && ${RM_CMD} -f ${with_zfs_attach}

mount_jail_fstab
export_jail_data_for_external_hook
external_exec_master_script "master_prestart.d"

if [ "${ver}" = "empty" ]; then
	#path="/"
	[ -z "${exec_start}" ] && exec_start="${jailsysdir}/${jname}/run.sh"
else
	[ ! -d "${data}/var/cache/pkg" ] && ${MKDIR_CMD} -p "${data}/var/cache/pkg"
fi

#determine that jail is FreeBSD. Useful for vnet operation in makejconf and
is_freebsd=0

if [ ${baserw} -eq 1 ]; then
	elftest=${data}/bin/sh
else
	elftest="${BASE_DIR}/bin/sh"
fi
[ -f "${elftest}" ] && osname=$( ${miscdir}/elf_tables --osname ${elftest} )

[ "${osname}" = "freebsd" ] && is_freebsd=1

# check for freshed version of the base via elf from /bin/sh
if [ ${is_freebsd} -eq 1 -a ${baserw} -eq 1 ]; then
	if [ -n ${BASE_DIR} -a -f "${BASE_DIR}/bin/sh" ]; then
		baseelf=$( ${miscdir}/elf_tables --ver ${BASE_DIR}/bin/sh 2>/dev/null )
		jailelf=$( ${miscdir}/elf_tables --ver ${elftest} 2>/dev/null )
		if [ ${baseelf} -gt ${jailelf} ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${BOLD}Notice: ${N1_COLOR}You have a more recent version of the base in ${BASE_DIR} (${N2_COLOR}${baseelf}${N1_COLOR}/${N2_COLOR}${jailelf}${N1_COLOR}).${N0_COLOR}"
			[ ${quiet} -ne 1 ] && ${ECHO} "${BOLD}Notice: ${N1_COLOR}Please consider upgrading jail base via ${N2_COLOR}cbsd jupgrade${N0_COLOR}"
		fi
	fi
fi

set -e
makejconf jname=${jname} out=${ftmpdir}/${jname}.conf ip6wa=${HAVE_IPV6} epair=${epairb_list} fbsd=${is_freebsd} quiet=${quiet}
set +e

#rctl/limits area
jrctl jname=${jname} mode=set quiet=${quiet}
[ -r "${jailsysdir}/${jname}/helpers/jrctl.sqlite" ] && nice=$( cbsdsqlro ${jailsysdir}/${jname}/helpers/jrctl.sqlite "SELECT cur FROM forms WHERE param=\"nice\"" )
[ -z "${nice}" ] && nice="0"
[ "${nice}" != "0" ] && [ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}jail renice: ${N2_COLOR}${nice}${N0_COLOR}"

# 2021-10: Lost relevance?
#unset_changes_20140930
#set_changes_20140930

etcupdate_check

[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Starting jail: ${N2_COLOR}${jname}, parallel timeout=${parallel}${N0_COLOR}"
TRAP="${TRAP} ${RM_CMD} -f ${ftmpdir}/jstart.${jname}.$$;"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

# check that the emulator is able to execute commands
res=$( emulator_exec_check )

ret=$?

if [ ${ret} -ne 0 ]; then
	[ ${quiet} -ne 1 ] && $ECHO "${N1_COLOR}emulator error: ${N2_COLOR}${res}${N0_COLOR}"
	jcleanup jname="${jname}"
	exit 0
fi

# TODO/NOTES:
# when path have no /var and/or /tmp (e.g empty fstab file), after jail start we have
# undesirable/undesirable tmpfs! bug/issue ?
# tmpfs on /usr/jails/jails/jail2/var (tmpfs, local)
# tmpfs on /usr/jails/jails/jail2/tmp (tmpfs, local)

CPUSET=
if [ -r "${jailsysdir}/${jname}/cpu" ]; then
	. ${jailsysdir}/${jname}/cpu
		CPUSET="${CPUSET_CMD} ${cpuset}"
	else
		if [ "${cpuset}" != "0" ]; then
			CPUSET="${CPUSET_CMD} ${cpuset}"
		fi
	fi

[ -r ${ftmpdir}/jstart.${jname}.err ] && ${RM_CMD} ${ftmpdir}/jstart.${jname}.err

case "${platform}" in
	DragonFly)
		# legacy jail tools only
		#cat ${ftmpdir}/${jname}.conf
		${MOUNT_CMD} -t devfs devfs ${path}/dev
		echo "[debug] ${JAIL_CMD} ${path} ${host_hostname} ${ip4_addr} ${exec_start}"
		/usr/local/cbsd/misc/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} ${path} ${host_hostname} ${ip4_addr} ${exec_start}
		ret=$?
		;;
	*)
		cbsdlogger NOTICE ${CBSD_APP}: ${CPUSET} /usr/local/cbsd/misc/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname}
		#echo "${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname}"
		case "${quiet}" in
			0)
				${CPUSET} /usr/local/cbsd/misc/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname}
				ret=$?
				;;
			1)
				${CPUSET} /usr/local/cbsd/misc/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname} > /dev/null
				ret=$?
				;;
		esac
esac

if [ ${ret} -ne 0 ]; then
	if [ -r ${ftmpdir}/jstart.${jname}.err ]; then
		${CAT_CMD} ${ftmpdir}/jstart.${jname}.err
		${RM_CMD} ${ftmpdir}/jstart.${jname}.err
	fi
	jcleanup jname=${jname}
	log_err 1 "${N1_COLOR}${CBSD_APP}: daemonize error${N0_COLOR}"
fi

trap "" HUP INT ABRT BUS TERM EXIT

[ -r "${ftmpdir}/jstart.${jname}.$$" ] && cbsd_pwait --pid=$( ${CAT_CMD} ${ftmpdir}/jstart.${jname}.$$ ) --timeout=${parallel} > /dev/null 2>&1
sleep 1

jstart_slow=0

# wait for jid
for i in 1 2 3 4 5 6; do
	get_jid
	ret=$?
	[ ${ret} -eq 1 -a -n "${myjid}" ] && break
	if [ "${quiet}" != "1" ]; then
		printf "${CLRLINE}"
		printf "${CURSORRST}"
		printf "${N1_COLOR}${CBSD_APP}: waiting for ${jname} JID: ${N2_COLOR}${i}/6 ... ${N0_COLOR}"
		jstart_slow=1
	fi
	sleep 1
done

if [ ${ret} -eq 0 ]; then
	[ "${quiet}" != "1" ] && ${ECHO} "${W1_COLOR}ERROR${N0_COLOR}"
	if [ -r ${ftmpdir}/jstart.${jname}.err ]; then
		${CAT_CMD} ${ftmpdir}/jstart.${jname}.err
		${RM_CMD} ${ftmpdir}/jstart.${jname}.err
	fi
	jcleanup jname=${jname}
	log_err 1 "${N1_COLOR}${CBSD_APP}: jail failed${N0_COLOR}"
fi

[ ${jstart_slow} -eq 1 -a "${quiet}" != "1" ] && ${ECHO} "${H1_COLOR}${myjid}${N0_COLOR}"
unset jstart_slow

### late start ###
# late exec_start: for zfsattached and vnet-based jails
# this is necessary for any operations that must be performed after
# creating the jail but before running real exec_start
late_start=

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
if [ -r ${with_zfs_attach} -a ${is_freebsd} -eq 1 ]; then
	[ -r ${mount_fstab} ] && attachzfs fstab=${mount_fstab} jname=${jname}
	[ -r ${mount_fstab}.local ] && attachzfs fstab=${mount_fstab}.local jname=${jname}
	# real exec_start for zfs attached jail
	# todo: FIB? NICE?
	${CPUSET} ${JEXEC_CMD} ${jname} ${ZFS_CMD} mount -a
	late_start="${exec_start}"
fi

# for VNET-based jail we also need special route
# due to jail.conf doesn't support for multiple NICs
if [ ${vnet} -eq 1 ]; then
	OIFS="${IFS}"
	IFS=","
	eth_seq=0
	int_iface=
	for i in ${epairb_list}; do
		[ -z "${i}" ] && continue
		IFS="${OIFS}"
		${IFCONFIG_CMD} ${i} up
		# we need setup lock here due to ethX name possible conflict with other jail
		_wait_count=0
		[ -z "${_iface_rename}" ] && _iface_rename=0

		if [ ${_iface_rename} -eq 1 ]; then
			while [ -r ${tmpdir}/eth${eth_seq}.locked ]; do
				_cur_time=$( ${DATE_CMD} +%s )
				eval $( ${STAT_CMD} -s ${tmpdir}/eth${eth_seq}.locked )
				_diff_time=$(( _cur_time - st_mtime ))

				if [ ${_diff_time} -gt 8 ]; then
					${IFCONFIG_CMD} eth${eth_seq} > /dev/null 2>&1
					_ret=$?
					if [ ${_ret} -ne 0 ]; then
						# remove orphaned lock
						${RM_CMD} -f ${tmpdir}/eth${eth_seq}.locked
						break
					fi
				fi

				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}jstart: eth${eth_seq} locked (${_diff_time}), waiting (${_wait_count}/12)..${N0_COLOR}"

				_wait_count=$(( _wait_count + 1 ))

				if [ ${_wait_count} -gt 12 ]; then
					_res=$( ${CAT_CMD} ${tmpdir}/eth${eth_seq}.locked )
					jstop jname=${jname} fast=1
					${IFCONFIG_CMD} ${i} destroy
					log_err 1 "${N1_COLOR}${CBSD_APP}: waiting for unlock: still locked by [${_res}]: ${N2_COLOR}${tmpdir}/eth${eth_seq}.locked${N0_COLOR}"
				fi

				sleep 1
			done

			# make lock
			echo "${jname}" > ${tmpdir}/eth${eth_seq}.locked
			${IFCONFIG_CMD} ${i} name eth${eth_seq} && ${IFCONFIG_CMD} eth${eth_seq} vnet ${jname}
			int_iface="eth${eth_seq}"
		else
			${IFCONFIG_CMD} ${i} vnet ${jname}
			int_iface="${i}"
		fi

		# more than one IPs in ip4_addr?
		multi_ip=0
		multi_ip4=0
		multi_ip6=0

		# duplicate code with cloudinit: todo: merge into generic func
		OIFS="${IFS}"
		IFS=","

		for _pureip in ${ip4_addr}; do
			IFS="${OIFS}"

			multi_ip=$(( multi_ip + 1 ))

			case "${_pureip}" in
				[Rr][Ee][Aa][Ll][Dd][Hh][Cc][Pp])
					# this file can be managed by users and/or 3rd party stuff (e.g. Reggae)
					[ -r ${workdir}/jails-data/${jname}-data/etc/rc.conf.d/network ] && . "${workdir}/jails-data/${jname}-data/etc/rc.conf.d/network"
					if [ -z "${dhclient_program}" ]; then
						# todo: FIB? NICE? FreeBSD-only environment?
						DHCLIENT_PROGRAM=`${CPUSET} ${JEXEC_CMD} ${jname} sysrc -n dhclient_program`
					else
						DHCLIENT_PROGRAM=${dhclient_program}
					fi
					# todo: FIB? NICE?
					${CPUSET} ${JEXEC_CMD} ${jname} ${DHCLIENT_PROGRAM} ${int_iface}
					;;
				0|[Dd][Hh][Cc][Pp])
					# just skip
					;;
				*)
					# alias ? mtu ?
					# we need to re-build ifconfig_${int_iface} vars if values already exist to merge new IP address and
					# and do not spoil the remaining data

					# Whats about Linux jail?
					if [ ! -r ${data}/etc/rc.conf ]; then
						if [ -x ${path}/bin/sh ]; then
							_osname=$( ${miscdir}/elf_tables --osname /bin/sh )
							# warning relevant for FreeBSD-based env
							if [ "${osname}" = "freebsd" ]; then
								${ECHO} "${W1_COLOR}warning: ${N1_COLOR}no such /etc/rc.conf: ${N2_COLOR}${name}${N1_COLOR}, skip for config ifconfig_${int_iface}${N0_COLOR}" 1>&2
							else
								# Linux env?
								if [ ! -r "${path}/etc/os-release" ]; then
									${ECHO} "${W1_COLOR}warning: ${N1_COLOR}no such /etc/os-release: ${N2_COLOR}${name}${N1_COLOR}, skip for config ifconfig_${int_iface}${N0_COLOR}" 1>&2
								else
									. ${path}/etc/os-release
									case "${ID}" in
										debian)
											${ECHO} "${N1_COLOR}${CBSD_APP} use debian helper for config_${int_iface}${N1_COLOR}"
											# just test, netlink/BSD wIP
											# mgmt for first boot only? ! -r ${data}/etc/network/interfaces.d/* ?
#											${CAT_CMD} > ${data}/etc/network/interfaces.d/lo <<EOF
#auto lo
#iface lo inet loopback
#EOF
#											${CAT_CMD} > ${data}/etc/network/interfaces.d/${int_iface} <<EOF
#auto ${int_iface}
#iface ${int_iface} inet static
#    address ${ip4_addr}/24
#    gateway 192.168.0.1
#
#EOF
											;;
										*)
											${ECHO} "${W1_COLOR}warning: ${N1_COLOR}skip config for ifconfig_${int_iface}: distri not supported yet: ${N2_COLOR}${ID}${N0_COLOR}" 1>&2
											;;
									esac
								fi
							fi
						fi
					else
						# FreeBSD-based env
						ifconfig_eth0=
						# build new ifconfig_eth0
						nomask=0
						strpos --str="${_pureip}" --search="/"
						nomask=$?
						mask=
						iptype ${_pureip}
						myip_type=$?
						# 1 - ipv4
						# 2 - ipv6
						# * - unknown
						case ${myip_type} in
							1)
								modif="inet"
								[ ${nomask} -eq 0 ] && mask="24"
								multi_ip4=$(( multi_ip4 + 1 ))
								;;
							2)
								modif="inet6"
								[ ${nomask} -eq 0 ] && mask="64"
								multi_ip6=$(( multi_ip6 + 1 ))
								;;
							*)
								modif=
								;;
						esac

						if [ -n "${modif}" ]; then
							if [ -n "${mask}" ]; then
								new_ifconfig_eth0="${modif} ${_pureip}/${mask}"
							else
								new_ifconfig_eth0="${modif} ${_pureip}"
							fi
						else
							# unknown IP type, pass as-is
							new_ifconfig_eth0="${_pureip}"
						fi

						case "${myip_type}" in
							1)
								if [ ${multi_ip4} -lt 2 ]; then
									find_ifconfig="ifconfig_${int_iface}"
								else
									alias_num=$(( multi_ip4 - 2 ))
									find_ifconfig="ifconfig_${int_iface}_alias${alias_num}"
								fi
								;;
							2)
								if [ ${multi_ip6} -lt 2 ]; then
									find_ifconfig="ifconfig_${int_iface}_ipv6"
								else
									alias_num=$(( multi_ip6 - 2 ))
									find_ifconfig="ifconfig_${int_iface}_ipv6_alias${alias_num}"
								fi
								;;
						esac

						eval $( ${GREP_CMD} ^${find_ifconfig}= ${data}/etc/rc.conf 2>/dev/null | ${HEAD_CMD} -n1 )
						if [ -n "${ifconfig_eth0}" ]; then
							${ECHO} "${N1_COLOR}adjust IP address for ${int_iface}: ${N2_COLOR}${ip4_addr}${N0_COLOR}" 1>&2
							for _part in ${ifconfig_eth0}; do
								[ "${_part}" = "inet" ] && continue
								[ "${_part}" = "inet6" ] && continue
								iptype ${_part}
								if [ $? -eq 1 ]; then
									#echo "!! IPv4 detected: ${_part}, skipp"
									continue
								fi
								if [ -z "${new_ifconfig_eth0}" ]; then
									new_ifconfig_eth0="${_part}"
								else
									new_ifconfig_eth0="${new_ifconfig_eth0} ${_part}"
								fi
							done
						fi
						/usr/local/cbsd/misc/cbsdsysrc -qf ${data}/etc/rc.conf ${find_ifconfig}="${new_ifconfig_eth0}" > /dev/null 2>&1
					fi
			esac
		done

		# release lock
		[ -r ${tmpdir}/eth${eth_seq}.locked ] && ${RM_CMD} -f ${tmpdir}/eth${eth_seq}.locked
		eth_seq=$(( eth_seq + 1 ))
		IFS=","
	done

	IFS="${OIFS}"

	[ -n "${oci_gw4}" ] && ci_gw4="${oci_gw4}"

	if [ -n "${ci_gw4}" ]; then
		OIFS="${IFS}"
		IFS=","

		for _pureip in ${ci_gw4}; do
			IFS="${OIFS}"

			case ${_pureip} in
				0)
					# do nothing
					;;
				auto)
					# always first?
					nic_address=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT nic_address FROM jailnic LIMIT 1 )
					# is ip?
					if [ -n "${nic_address}" -a "${nic_address}" != "0" ]; then
						echo "${N1_COLOR}${CBSD_APP}: set defaultrouter to nic_address: ${N2_COLOR}[${nic_address}]${N0_COLOR}"
						/usr/local/cbsd/misc/cbsdsysrc -qf ${data}/etc/rc.conf defaultrouter="${nic_address}" > /dev/null 2>&1
					elif [ -n "${oci_gw4}" ];then
						nic_address="${oci_gw4}"
						echo "${N1_COLOR}${CBSD_APP}: set defaultrouter to by config file: ${N2_COLOR}[${nic_address}]${N0_COLOR}"
						/usr/local/cbsd/misc/cbsdsysrc -qf ${data}/etc/rc.conf defaultrouter="${nic_address}" > /dev/null 2>&1
					else
						echo "${N1_COLOR}${CBSD_APP}: ci_gw4 sets to auto, but nic_address is empty${N0_COLOR}"
					fi
					;;
				*)
					iptype ${_pureip}
					myip_type=$?
					# 1 - ipv4
					# 2 - ipv6
					# * - unknown
					case ${myip_type} in
						1)
							/usr/local/cbsd/misc/cbsdsysrc -qf ${data}/etc/rc.conf defaultrouter="${_pureip}" > /dev/null 2>&1
							;;
						2)
							/usr/local/cbsd/misc/cbsdsysrc -qf ${data}/etc/rc.conf ipv6_defaultrouter="${_pureip}" > /dev/null 2>&1
							;;
						*)
							;;
					esac
					;;
			esac
		done
	fi

	IFS="${OIFS}"
	late_start="${exec_start}"
fi

if [ -n "${late_start}" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}late_start in progress...${N2_COLOR}"
	# todo: FIB? NICE?
	${CPUSET} ${JEXEC_CMD} ${jname} ${late_start}
fi
### /late start ###

# update state_time
cbsdsqlrw local UPDATE jails SET state_time="(strftime('%s','now'))" WHERE jname='${jname}'

# make id file
UNDHOSTNAME=$( echo ${hostname} | ${TR_CMD}  "." "_" )
FID="/var/run/jail_${UNDHOSTNAME}.id"
echo ${ST} > ${FID}

# todo: CPUSET, NICE, FIB?
exec_cbsdjail_first_boot
external_exec_script -s start.d

get_jid
jid=${myjid}

if [ "${platform}" = "DragonFly" ]; then
	[ ${allow_raw_sockets} -eq 1 ] && ${SYSCTL_CMD} -w jail.${jid}.net_raw_sockets=1
fi

external_exec_master_script "master_poststart.d"

if [ ${myjid} -gt 0 ]; then
	status="1"
	# make shared lock
	echo "${nodename}" > ${jaillock}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jaillock}
else
	status="0"
	# looks like jail start is failed, execute post script
	exec_master_poststop
	external_exec_master_script "master_poststop.d"
	exec_poststop
	jcleanup jname=${jname}

	# redis
	if [ "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
		cbasredis hset "jail:${jname}" status -1 || echo "WARNING: Failed to update Redis"
		cbasredis hdel "jail:${jname}" jid || echo "WARNING: Failed to update Redis"
		cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":-1}'
	fi
fi

cbsdsqlrw local "UPDATE jails SET jid=${myjid},status=${status} where jname='${jname}'"
${RM_CMD} -f ${ftmpdir}/${jname}.conf
fwcounter
expose mode=apply

if [ ${myjid} -gt 0 ]; then
	if [ -r "${jailsysdir}/${jname}/cpu" ]; then
		. ${jailsysdir}/${jname}/cpu
		echo "${CPUSET_CMD} ${cpuset} -j ${myjid}" 1>&2
		${CPUSET_CMD} ${cpuset} -j ${myjid}
	else
		if [ "${cpuset}" != "0" ]; then
			echo "${CPUSET_CMD} ${cpuset} -j ${myjid}" 1>&2
			${CPUSET_CMD} ${cpuset} -j ${myjid}
		fi
	fi
fi

# VNC auto start
if [ -x "${distmoduledir}/vncterm.d/cbsdvnc" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}vncterm is installed, launch vncterm session...${N0_COLOR}"
	/usr/local/cbsd/misc/daemonize /usr/local/bin/cbsd vncterm jname=${jname} mode=run addr=127.0.0.1 >/dev/null 2>&1
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_jail_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_jail_queue_name} id=${jname} cmd=jstart status=2 data_status=1 workdir="${workdir}"
fi

# API compat: ~cbsd/jails-system/info*.*
/usr/local/cbsd/misc/daemonize /usr/local/cbsd/tools/save-jail-info jname=${jname}

end_time=$( ${DATE_CMD} +%s )
diff_time=$(( end_time - st_time ))
# redis
if [ ${myjid} -gt 0 -a "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
	cbsdredis hset "jail:${jname}" status 2 jid ${myjid} started ${st_time} echo "WARNING: Failed to update Redis"
	cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":0, "duration":'${diff_time}'}'
fi
diff_time=$( displaytime ${diff_time} )

if [ -f "${jailsysdir}/${jname}/jail-message" -a ${quiet} -ne 1 ]; then
	${ECHO} "${N1_COLOR}---- ${N2_COLOR}jail messages ${N1_COLOR}----${N0_COLOR}" 1>&2
	# todo: trim special/carp/interface value?
	${SED_CMD} -e "s|\${ip4_addr}|${ip4_addr}|g" "${jailsysdir}/${jname}/jail-message" 1>&2
	${ECHO} "${N1_COLOR}---- ${N2_COLOR}jail messages ${N1_COLOR}----${N0_COLOR}" 1>&2
fi

[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CBSD_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"
cbsdlogger NOTICE ${CBSD_APP}: jail ${jname} started in ${diff_time}

exit 0
