#!/usr/local/bin/cbsd
MYARG=""
MYOPTARG="jname node full inter incr aincr incrcnt aremove version checkonly"
MYDESC="ZFS migrator for CBSD"
CBSDMODULE="sys"
EXTHELP="wf_zfs_migrator"
ADDHELP="
${H3_COLOR}Description:${N0_COLOR}

Alternative virtual environmental migration, based on the ZFS file system and its
capabilities. To transfer the virtual container, you must first add the destination node 
via 'cbsd node mode=add' command.

${H3_COLOR}Options:${N0_COLOR}

 ${N2_COLOR}full=${N0_COLOR}      - set '1' to full migration cycle - j2prepare, sync, jswmode. ( 1 -default );
 ${N2_COLOR}inter=${N0_COLOR}     - set '0' to prevent any questions and to accept answers by default;
 ${N2_COLOR}incr=${N0_COLOR}      - set '1' to perform an incremental migration (default is 0);
 ${N2_COLOR}aincr=${N0_COLOR}     - set '1' to perform automatic incremental mode (default is 0);
 ${N2_COLOR}aremove=${N0_COLOR}   - possible values: '0', '1' ,'3':
              - if aremove=0, don't remove source instance after migration without asking;
              - if aremove=1, auto remove source instance after migration without asking;
              - if aremove=3 (by default), and inter not 0, show yesno prompt for interactive choice;
 ${N2_COLOR}version=${N0_COLOR}   - set '1' to show zfs-migrator version only;
 ${N2_COLOR}checkonly=${N0_COLOR} - set '1' to test node for zfs-migrator compatible, errcode=1 when not and errcode=0 when ok;

${H3_COLOR}Examples:${N0_COLOR}

 # cbsd zfs-migrator jname=xxx node=fbsd2.my.domain

 # For soft migration ( 1 full + stop env + incement ):
 # cbsd zfs-migrator jname=xxx node=zfsmigrat2.my.domain aincr=1

${H3_COLOR}See also:${N0_COLOR}

 # cbsd node --help
 # cbsd j2prepare --help
 # cbsd jrclone --help
 # cbsd jcoldmigratee --help

"

. ${subrdir}/nc.subr

clear_stfile()
{
	if [ "x${STFILE}" != "x" ]; then
		if [ -f ${STFILE} ]; then
			${ECHO} "${N1_COLOR}* ${N2_COLOR}Clearing state file: ${DC_NAME}:${STFILE}${N0_COLOR}"
			${RM} ${STFILE} > /dev/null 2>&1
		fi
	fi
}

inst_stop () {
	${ECHO} "${N1_COLOR}**** ${N2_COLOR}Runtime impact and pre-migration changes to ${emulator} instance${N0_COLOR}"
	${ECHO} "${N1_COLOR}**** ${N2_COLOR}${jname} (${host_hostname})${N0_COLOR}"
	if [ "${jail_status}" = "running" ]; then
		${ECHO} "${N1_COLOR}* ${N2_COLOR}Instance ${fqdn} is running; shutting it down, please wait${N0_COLOR}"
		case "${emulator}" in
			jail)
				jstop jname=${jname} > /dev/null 2>&1
				;;
			bhyve)
				bstop jname=${jname} > /dev/null 2>&1
				;;
		esac
		INST_STATE="stopped"
		${ECHO} "${N2_COLOR}Done, new state is ${INST_STATE}"
	else
		${ECHO} "${N1_COLOR}* ${N2_COLOR}Instance ${jname} is already shutdown.  Proceeding.${N0_COLOR}"
	fi
}

# function
fsNotExist() {
	local dset=$1
	${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} list ${dset%@*} 2>/dev/null >/dev/null
	return $?
}

# function
transferError() {
	echo
	echo "      > transfer failed!"
	clear_stfile
	exit 24
}

# function
transferQuota() {
	local dset=$1
	shift
	local dsetQuota=$( ${SSH_CMD} -q -t ${remote_node_ip} "${ZFS_CMD} get -Hp -o value quota ${dset}" | ${TR_CMD} -d '[[:space:]]' )
	if [ ${dsetQuota} -eq 0 ]; then
		return;
	fi

	local dsetSpace=$( ${SSH_CMD} -q -t ${remote_node_ip} "${ZFS_CMD} list -Hp -o used ${dset}" | ${TR_CMD} -d '[[:space:]]' )
	local freeSpace=$((${dsetQuota} - ${dsetSpace}))
	local gigabyte=$((1024 * 1024 * 1024))
	local quota_bump_msg=""

	if [ ${freeSpace} -lt ${gigabyte} ]; then
		dsetQuota=$((${dsetQuota} + ${gigabyte} * ${bump_size}))
		quota_bump_msg="(+${bump_size}Gb)"
	fi

	${SSH_CMD} -q -t ${remote_node_rip} ${ZFS_CMD} set quota=${dsetQuota} ${dset} && printf " quota${quota_bump_msg}..." || transferError
}

# function()
transferOptions() {
	local dset=$1
	shift
	local fields=$( echo $@ | sed 's/ /,/g' )

	local opts=$( ${SSH_CMD} -q -t ${remote_node_ip} "${ZFS_CMD} get -Hp -o property,value ${fields} ${dset}" | ${AWK_CMD} '{if ($2 !~ /^(none|-)/) printf $1"="$2}' )

	if [ ! -z ${opts} ]; then
		printf "    + Transferring dataset options to ${dset}..."
		for opt in ${opts};do
			if [ ${opt%=*} = "quota" ]; then
				transferQuota ${dset}
			else
				${SSH_CMD} -q -t ${remote_node_rip} ${ZFS_CMD} set ${opt} ${dset} && printf " ${opt}..." || transferError
			fi
		done
		echo " done..."
	fi

}

# function
transferAll() {
	local dset=$1
	local origin=$(${SSH_CMD} -q -t ${remote_node_ip} "${ZFS_CMD} get origin ${dset%@*}" | ${TAIL} -n 1 | ${AWK} '{printf $3}')
	if [ "x${origin}" != "x-" ]; then
		fsNotExist ${origin}
		if [ $? -ne 0 ];then
			transferAll ${origin}
		fi
		fsNotExist ${dset}
		if [ $? -ne 0 ];then
			echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
			_ret=$?
			if [ ${_ret} -eq 0 ]; then
				${ZFS_CMD} send -I ${origin} ${dset} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT}/${jname} 2>/dev/null && echo "done" || transferError
			else
				${ZFS_CMD} send -I ${origin} ${dset} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT} 2>/dev/null && echo "done" || transferError
			fi
		fi
	else
		echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
		_ret=$?
		if [ ${_ret} -eq 0 ]; then
			${ZFS_CMD} send -p ${dset} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT}/${jname} 2>/dev/null && echo "done" || transferError
		else
			${ZFS_CMD} send -p ${dset} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT} 2>/dev/null && echo "done" || transferError
		fi
	fi

	transferOptions ${dset%@*} quota
}


# full migration cycle
full=1
# interactive
inter=1
ALWAYS_YES=1

# Incremental by default is 0
incr=0

# incrCnt not implemented yet
#incrCnt

# Automatic Incremental by default is 0
aincr=0

# Automatic remove by default: ask user's
aremove=3

# ZFS_CMD Migrator version
ZMver="1"

# Checkonly?
checkonly=0

. ${cbsdinit}
. ${system}

[ "${version}" = "1" ] && err 0 "${ZMver}"

if [ "${checkonly}" = "1" ]; then
	export NOCOLOR=1
	[ ${zfsfeat} -ne 1 ] && err 1 "${N1_COLOR}zfsfeat disabled, zfs-migrator non-working here${N0_COLOR}"
	LOCAL_ZROOT=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} 2>/dev/null | ${HEAD_CMD} -n 1 )
	[ -z "${LOCAL_ZROOT}" ] && err 1 "${N1_COLOR}${CBSD_APP}: error: unable to determine ZROOT for datadir, zfs-migrator non-working here: ${N2_COLOR}${ZFS_CMD} get -Ho value name ${jaildatadir}${N0_COLOR}"
	err 0 "Ready"
fi

if [ ${inter} -eq 1 ]; then
	[ -z "${jname}" ] && err 1 "${N1_COLOR}Empty jname=${N0_COLOR}"
	if [ -z "${node}" ]; then
		${ECHO} "${N1_COLOR}Empty node=\nAvailable nodes:${N0_COLOR}"
		node mode=list display=nodename header=0
		exit 1
	fi
fi

# ZFS_CMD only
case ${zfsfeat} in
	1)
		. ${subrdir}/zfs.subr
		;;
	*)
		err 1 "${N1_COLOR}zfs-migrator for ZFS_CMD-based system only. Current host has zfsfeat is ${N2_COLOR}${zfsfeat}${N0_COLOR}"
esac

. ${subrdir}/bhyve.subr

#
# set some vars:
BASENAME='/usr/bin/basename'
scriptName=`${BASENAME} ${0}`
AWK='/usr/bin/awk'
BC='/usr/bin/bc'
CAT='/bin/cat'
CHOWN='/usr/sbin/chown'
CHMOD='/bin/chmod'
CUT='/usr/bin/cut'
CURL='/usr/local/bin/curl'
CURL_OPTS='--connect-timeout 10 -sS -i -H accept:application/json -H content-type:application/json'
STTIME=`${DATE_CMD} +%s`
BEGTIME=""
DTIME=""
ETIME=""
ENDTIME=""
TDIFF=""
DIFFT=""
IMGADM='/usr/sbin/imgadm'
JSON='/usr/bin/json'
TAIL='/usr/bin/tail'
KEYGEN='/usr/bin/ssh-keygen'
LS='/bin/ls'
MV='/bin/mv'
NDD='/usr/sbin/ndd'
RM='/bin/rm'
SCP='/usr/bin/scp'
SED='/usr/bin/sed'
SVCADM='/usr/sbin/svcadm'
UCONF='/usbkey_config'
KEYPATH="/root/.ssh/migr-key-$$"
LOGDIR="/var/tmp"
LOGFILE=""
STFILE=""
AUTHFILEBK="/root/.ssh/authorized_keys.$$"
KEYCHK=0	# if != 0, ssh key in place; if = 0, password prompts
OPTSSH_CMD="-q"	# if KEYCHK != 0, sets opt '-q -i KEYPATH'
MDSSZ=0
OIFS=${IFS}
IpAdDr=""
cntIncr=0
incrCnt=""
cntPrev=""
prevCnt=""

case "${incr}" in
	0)
		migrType="Non-Incremental"
		;;
	1)
		migrType="Incremental"
		;;
esac

case "${aincr}" in
	1)
		incr=1
		aincr=1
		migrType="automatic-Incremental"
		;;
esac

# If instance is over quota, how many GB will the quota be increased so the
# transfer can succeed? Must be an integer.
bump_size="1"

DC_NAME="datacenter1"

CBSD_MIGRNAME='1504213200_cbsd_migration_1504213200'
IMGCHK=0		# 0:  value of DS origin = '-'; we're doing a standard dataset transfer
			# 1:  value of DS origin = 'ZROOT/:IMGJNAME@final; test for img on dest CN,
			#     if !exist, try to import, if import fail, zfs send img followed
			#     by incr roll up of the instance datasets

IMGSND=3		# 0:  origin DS already exists on dest, no need for import
			# 1:  unset and irrelevant
			# 2:  origin DS ! exists on dest, try to import
			# 3:  origin DS relation to instance lost between src and dest, don't try to import

dsCNT=1
DSORIG=0

if [ ${incr} -gt 0 -a "x${incrCnt}" != "x" ]; then
	cntIncr=${incrCnt}
	cntPrev=$(( incrCnt - 1 ))
fi

# Collect our initial information
${ECHO} "\n           ${N1_COLOR}-: Welcome to the CBSD ZFS_CMD Migrator :-${N0_COLOR}"

if [ -z "${jname}" ]; then
	while [ true ]; do

		read -p "Name of Instance to Move ( ? - to list, 0 - cancel ): " jname

		[ "${jname}" = "0" ] && exit 0

		if [ "${jname}" = "?" ]; then
			echo -n "jails: " && jorder
			echo -n "bhyve: " && border
			continue
		fi

		vm_status=$( jstatus jname=${jname} )
		ret=$?
		[ ${ret} -eq 1 ] && break

		${ECHO} "${N1_COLOR}No such environment here: ${N2_COLOR}${jname}${N0_COLOR}"
	done
fi

jstatus jname=${jname} > /dev/null 2>/dev/null
[ $? -eq 0 ] && err 1 "${N1_COLOR}No such environment here: ${N2_COLOR}${jname}${N0_COLOR}"

if [ -z "${node}" ]; then
	while [ true ]; do
		read -p "Destination Nodename ( ? - to list, 0 - cancel ): " node
		[ "${node}" = "0" ] && exit 0

		if [ "${node}" = "?" ]; then
			echo -n "nodes: " && mode=list display=nodename header=0 | ${XARGS_CMD}
			continue
		fi

		test_for_node=$( ${SQLITE3_CMD} ~cbsd/var/db/nodes.sqlite "SELECT nodename FROM nodelist WHERE nodename='${node}'" )
		[ "${test_for_node}" = "${node}" ] && break
		${ECHO} "${N1_COLOR}No such node here: ${N2_COLOR}${node}${N0_COLOR}"
	done
	echo
fi

if [ ${incr} -gt 0 ]; then
	if [ -d ${LOGDIR} ]; then
		LOGFILE="${LOGDIR}/migrator-${jname}-${node}-${STTIME}.log"
		STFILE="${LOGDIR}/migrator-${jname}-${node}-${STTIME}.st"
		${TOUCH_CMD} ${LOGFILE} > /dev/null 2>&1
		if [ $? -eq 0 ]; then
			${TOUCH_CMD} ${STFILE}
			exec 2>&1 |tee -a ${LOGFILE}
			exec 1>&1 |tee -a ${LOGFILE}
			if [ -f ${STFILE} ]; then
				SFCHK=0
				${ECHO} "${N1_COLOR}* ${N2_COLOR}Setting state file for removal by CBSD...${N0_COLOR}"
				${ECHO} "${N2_COLOR}    + chown cbsd:cbsd ${STFILE}${N0_COLOR}"
				${CHOWN} cbsd:cbsd ${STFILE}
				if [ $? -ne 0 ]; then
					${ECHO} "${N2_COLOR}      !! Root privileges required to remove state file: ${STFILE}${N0_COLOR}"
					SFCHK=$(( SFCHK + 1 ))
				else
					${ECHO} "${N2_COLOR}    + chmod 664 ${STFILE}${N0_COLOR}"
					${CHMOD} 664 ${STFILE}
					if [ $? -ne 0 ]; then
						${ECHO} "${N2_COLOR}      !! Root privileges required to remove state file: ${STFILE}${N0_COLOR}"
						SFCHK=$(( SFCHK + 1 ))
					fi
				fi
				if [ ${SFCHK} -eq 0 ]; then
					${ECHO} "${N2_COLOR}  - State file modified.${N0_COLOR}"
				else
					${ECHO} "${N2_COLOR}  - State file could not be modified!${N0_COLOR}"
				fi
			else
				err 1 "${N1_COLOR}State file does not exist: ${N2_COLOR}${STFILE}${N0_COLOR}"
			fi
		else
			err 1 "${N1_COLOR}Could not write log and write files: ${N2_COLOR}${LOGFILE},${STFILE}${N0_COLOR}"
		fi
	fi
fi

${ECHO} "${N1_COLOR}* ${N2_COLOR}Gathering instance and nodes data....${N0_COLOR}"
printf "${N2_COLOR}    + retrieving instance date... ${N0_COLOR}"

_res=$( rexe node=${node} tryoffline=1 date )
_ret=$?
if [ ${_ret} -ne 0 ]; then
	err 1 "${N1_COLOR}connection to remote host failed: ${_res}${N0_COLOR}"
fi
${ECHO} "${H2_COLOR}${_res}${N0_COLOR}"

# check that remote node is ZFS_CMD
d_migrator_ready=$( rexe tryoffline=1 node=${node} '/usr/local/bin/cbsd zfs-migrator checkonly=1' | ${AWK_CMD} '{printf $1}' | ${TR_CMD} -d '\r\n' )
_ret=$?
if [ ${_ret} -ne 0 ]; then
	err 1 "${N1_COLOR}Remote node is not ready ( /usr/local/bin/cbsd zfs-migrator checkonly=1 ): ${d_migrator_ready}${N0_COLOR}"
fi
[ "${d_migrator_ready}" != "Ready" ] && err 1 "${N1_COLOR}Remote node is not ready ( ${SSH_CMD} -q ${remote_node_rip} /usr/local/bin/cbsd zfs-migrator checkonly=1 ): [${d_migrator_ready}]${N0_COLOR}"

. ${subrdir}/rcconf.subr

myjid=$( cbsdsqlro local "SELECT jid FROM jails WHERE jname='${jname}'" 2>/dev/null )

case ${myjid} in
	0)
		jail_status="stopped"
		;;
	*)
		jail_status="running"
		;;
esac

# todo: check for IP addrs reserved
IP_ARR="${ip4_addr}"

${ECHO} "${N2_COLOR}    + retrieving node hostname, CBSD version...${N0_COLOR}"
if [ -r ~cbsd/nodename ]; then
	my_nodename=$( ${CAT_CMD} ~cbsd/nodename | ${AWK_CMD} '{printf $1}' )
else
	err 1 "${N1_COLOR}${CBSD_APP}: no nodename?: ${N2_COLOR}~cbsd/nodename${N0_COLOR}"
fi
[ -z "${my_nodename}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no nodename?: ${N2_COLOR}~cbsd/nodename | ${AWK_CMD} '{printf $1}'${N0_COLOR}"

remote_node_ip=$( getinfo mode=quiet nodeip )
remote_node_name="${node}"
remote_node_rip=$( ${SQLITE3_CMD} ~cbsd/var/db/nodes.sqlite "SELECT ip FROM nodelist WHERE nodename='${node}'" 2>/dev/null )

[ -z "${remote_node_rip}" ] && err 1 "${N1_COLOR}Could not identify IP address for node: ${N2_COLOR}${node}${N0_COLOR}"
source_vm_name="${jname}"
dest_vm_name="${jname}"


# todo: alternative jname on dst?
printf "${N2_COLOR}    + checking for datasets...${N0_COLOR}"

if [ -r ${tmpdir}/.vmmigrator.$$ ]; then
	${RM_CMD} -f ${tmpdir}/.vmmigrator.$$
fi

LOCAL_ZROOT=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} 2>/dev/null )
[ -z "${LOCAL_ZROOT}" ] && err 1 "${N1_COLOR}Unable to determine local zroot${N0_COLOR}"

# TODO: multi-environment support via ARGS
REMOTE_ZROOT=$( rexe tryoffline=1 node=${node} '. /etc/rc.conf && . ${cbsd_workdir}/cmd.subr && ${ZFS_CMD} get -Ho name name ${cbsd_workdir}/jails-data' | ${TR_CMD} -d '\r\n' )
_ret=$?
if [ ${_ret} -ne 0 ]; then
	err 1 "${N1_COLOR}Unable to determine remote zroot, check SSH_CMD or ZFS_CMD feature on remote host [${ZFS_CMD} get -Ho name name ${cbsd_workdir}/jails-data]: ${REMOTE_ZROOT}${N0_COLOR}"
fi
[ -z "${REMOTE_ZROOT}" ] && err 1 "${N1_COLOR}Unable to determine remote zroot, check SSH_CMD or ZFS_CMD feature on remote host${N0_COLOR}"

${ECHO} "${H2_COLOR}${REMOTE_ZROOT}${N0_COLOR}"

REMOTE_WORKDIR=$( rexe tryoffline=1 node=${node} '. /etc/rc.conf && echo ${cbsd_workdir}' | ${TR_CMD} -d '\r\n' )
_ret=$?
if [ ${_ret} -ne 0 ]; then
	err 1 "${N1_COLOR}Unable to determine remote workdir}${N0_COLOR}"
fi
[ -z "${REMOTE_ZROOT}" ] && err 1 "${N1_COLOR}Unable to determine remote workdir}${N0_COLOR}"

case "${emulator}" in
	jail)
		${ZFS_CMD} list -Hpro name,usedbydataset -t filesystem,volume ${LOCAL_ZROOT} | ${GREP_CMD} ${LOCAL_ZROOT}/${jname} 2>/dev/null >> ${tmpdir}/.vmmigrator.$$
		;;
	bhyve)
		${ZFS_CMD} list -Hpro name,usedbydataset -t filesystem,volume ${LOCAL_ZROOT} | ${GREP_CMD} "${LOCAL_ZROOT}/${jname}" 2>/dev/null | ${GREP_CMD} -v "dsk[0-9].vhd" >> ${tmpdir}/.vmmigrator.$$
		${ZFS_CMD} list -Hpro name,usedbydataset -t filesystem,volume ${LOCAL_ZROOT} | ${GREP_CMD} "${LOCAL_ZROOT}/${jname}/dsk[0-9].vhd" 2>/dev/null >> ${tmpdir}/.vmmigrator.$$
		;;
esac

echo "TMP:"
cat  ${tmpdir}/.vmmigrator.$$

if [ -s ${tmpdir}/.vmmigrator.$$ ]; then
	MDSSZ=`${AWK} 'BEGIN {tot=0}; {tot+=$2}; END {printf tot}' ${tmpdir}/.vmmigrator.$$`
fi

_num_of_dataset=$( ${WC_CMD} -l ${tmpdir}/.vmmigrator.$$ | ${AWK_CMD} '{printf $1}' )

if [ "${_num_of_dataset}" = "0" ]; then
	${RM_CMD} -f "${tmpdir}/.vmmigrator.$$"
	${ECHO} "${N1_COLOR}${CBSD_APP}: error: no such datasets for jname: ${N2_COLOR}${jname}${N0_COLOR}"
	case "${emulator}" in
		jail)
			echo "${ZFS_CMD} list -Hpro name,usedbydataset -t filesystem,volume ${LOCAL_ZROOT} | ${GREP_CMD} ${LOCAL_ZROOT}/${jname}"
			;;
		bhyve)
			echo "${ZFS_CMD} list -Hpro name,usedbydataset -t filesystem,volume ${LOCAL_ZROOT} | ${GREP_CMD} \"${LOCAL_ZROOT}/${jname}/dsk[0-9].vhd\""
			;;
	esac
	exit 1
fi

rise="Bytes"
SZMDS=0
if [ 0${MDSSZ} -lt 1024 ]; then
	SZMDS=${MDSSZ}
elif [ 0${MDSSZ} -lt 1048576 ]; then
	SZMDS=`echo "scale=2; ${MDSSZ}/1024" | ${BC}`
	rise="KBytes"
elif [ 0${MDSSZ} -lt 1073741824 ]; then
	SZMDS=`echo "scale=2; ${MDSSZ}/1048576" | ${BC}`
	rise="MBytes"
elif [ 0${MDSSZ} -lt 1099511627776 ]; then
	SZMDS=`echo "scale=2; ${MDSSZ}/1073741824" | ${BC}`
	rise="GBytes"
elif [ 0${MDSSZ} -lt 1125899906842624 ]; then
	SZMDS=`echo "scale=2; ${MDSSZ}/1099511627776" | ${BC}`
	rise="TBytes"
fi

miti="seconds"
mav=`echo "scale=2; ((${MDSSZ}/1048576)/20)+120" | ${BC}`
if [ 0${MDSSZ} -gt 1048576 ]; then
	if [ 0`echo "${mav}" | ${CUT} -d\. -f1` -gt 3600 ]; then
		mav=`echo "scale=2; ${mav}/3600" | ${BC}`
		miti="hours"
	elif [ 0`echo "${mav}" | ${CUT} -d\. -f1` -gt 60 ]; then
		mav=`echo "scale=2; ${mav}/60" | ${BC}`
		miti="minutes"
	fi
fi

# node key
if [ ${KEYCHK} -ne 0 ]; then
	OPTSSH_CMD="-q -i ${KEYPATH}"
fi

# todo: make retrinv and obtain information from SQLite directory ?
d_cbsd_ver=$( rexe tryoffline=1 node=${node} '. /etc/rc.conf && . ${cbsd_workdir}/cmd.subr && /usr/local/bin/cbsd -c version 2>/dev/null | ${AWK_CMD} "{printf $1}"' | ${TR_CMD} -d '\r\n' )
_ret=$?
if [ ${_ret} -ne 0 ]; then
	err 1 "${N1_COLOR}Unable to determine remote CBSD version: ${d_cbsd_ver}${N0_COLOR}"
fi
[ -z "${d_cbsd_ver}" ] && err 1 "${N1_COLOR}Unable to determine remote CBSD version${N0_COLOR}"
s_cbsd_ver=$( version )

if [ "${s_cbsd_ver}" != "${d_cbsd_ver}" ]; then
	err 1 "${N1_COLOR}CBSD version not equal! migration is supported only on identical versions: local[${s_cbsd_ver}], remote[${d_cbsd_ver}]}${N0_COLOR}"
fi

if [ ${incr} -ne 1 ]; then
	d_instance_exist=$( rexe tryoffline=1 node=${node} "/usr/local/bin/cbsd jstatus jname=${jname}" )
	ret=$?
	[ ${ret} -ne 0 ] && err 1 "${N1_COLOR}Node ${node} already has instance named?: ${N2_COLOR}${jname}${N0_COLOR}"
fi

${ECHO} "${N2_COLOR}  - Data gathering complete!${N0_COLOR}"
srcpad=" "
destpad=" "
${CAT_CMD} <<EOF

We will be migrating:
     INSTANCE:
               jname:  ${jname}
                fqdn:  ${host_hostname}
          IP Addr(s):  ${ip4_addr}
          datacenter:  ${DC_NAME}
      instance state:  ${jail_status}
                type:  ${emulator}
               owner:  root
           create at:  -
          base image:  -
  total dataset size:  ${SZMDS} ${rise} across ${dsCNT} datasets
        migration id:  $$
 est. migration time:  ${mav} ${miti} (@ ~20 MB / second (~1.17 GB / minute); +2 minutes extra)
      migration type:  ${migrType}
EOF
if [ ${incr} -gt 0 ]; then
	cat <<EOF
  migration log file:  ${DC_NAME}:${LOGFILE}
migration state file:  ${DC_NAME}:${STFILE}
EOF
fi

cat <<EOF

                    Source                                        Destination
----------------------------------------------  ----------------------------------------------
EOF
printf "Host:     %-36s  Host:     %-36s\n" ${my_nodename} ${remote_node_name}
printf "JNAME:    %-36s  JNAME:    %-36s\n" ${source_vm_name} ${dest_vm_name}
printf "SDC Ver:  %-36s  SDC Ver:  %-36s\n" ${s_cbsd_ver} ${d_cbsd_ver}
printf "IP Addr:  %-36s  IP Addr:  %-36s\n" ${remote_node_ip} ${remote_node_rip}
printf "ZFS:      %-36s  ZFS:      %-36s\n" ${LOCAL_ZROOT} ${REMOTE_ZROOT}
printf "API ver:  %-36s  API ver:  %-36s\n" ${s_cbsd_ver} ${d_cbsd_ver}

echo

if getyesno "Are you ready to proceed? "; then
	echo
else
	${ECHO} "${N1_COLOR}Exiting.${N0_COLOR}"
	clear_stfile
	exit 1
fi

BEGTIME=`${DATE_CMD}`
STTIME=`${DATE_CMD} +%s`

${ECHO} "${N1_COLOR}* ${N2_COLOR}Checking for origin instance dataset...${N0_COLOR}"

if [ "${emulator}" = "bhyve" ]; then
	DSORIG=`${ZFS_CMD} get -Ho value origin ${LOCAL_ZROOT}/${jname}/dsk1.vhd`
else
	DSORIG=`${ZFS_CMD} get -Ho value origin ${LOCAL_ZROOT}/${jname}`
fi

# check if origin dataset exists and is ${REMOTE_ZROOT}/...@final, ignore otherwise
if [ "x${DSORIG}" = "x${REMOTE_ZROOT}/${jname}@final" ]; then
	IMGCHK=1
	echo "    + origin dataset is ${DSORIG}"
	DSORIG=`echo "${DSORIG}" | ${SED} -e 's;^${LOCAL_ZROOT}/;;'`
#	DSOBJ=(`echo "${DSORIG}" | ${SED} -e 's;@; ;'`)
	DSOBJ=$( echo "${DSORIG}" | ${SED} -e 's;@; ;' )
	echo "DSOBJ stop: [${DSOBJ}]"
	exit 0
	# DSOBJ:
	#    0:  image jname derived from dataset name (${LOCAL_ZROOT}/IMG_JNAME@something)
	#    1:  snapshot name (part that trails @)
else
	${ECHO} "${N2_COLOR}    + origin dataset null or non-standard (${DSORIG})${N0_COLOR}"
fi

# in generic node install, $DSORIG = "-"
if [ ${IMGCHK} -ne 0 ]; then
#	${SSH_CMD} -q ${remote_node_rip} ${ZFS_CMD} list -Ho name ${REMOTE_ZROOT}/${DSORIG} >/dev/null 2>&1
	rexe tryoffline=1 node=${node} ". /etc/rc.conf && . \${cbsd_workdir}/cmd.subr && \${ZFS_CMD} list -Ho name ${REMOTE_ZROOT}/${DSORIG} >/dev/null 2>&1"
	if [ $? -ne 0 ]; then
		# bhyve?
		${SSH_CMD} -q ${remote_node_rip} ${ZFS_CMD} list -Ho name ${REMOTE_ZROOT}/${jname} >/dev/null 2>&1
		if [ $? -ne 0 ]; then
			IMGSND=2	# origin DS and base image of it don't exist, will try import
			${ECHO} "${N2_COLOR}      > origin DS doesn't exist on ${remote_node_name}, will need to try import${N0_COLOR}"
			${ECHO} "${N2_COLOR}        of origin DS (${DSORIG})${N0_COLOR}"
		else
			IMGSND=3	# origin DS base exists, but not snap the instance was
					#   create from, since we cannot just send the snapshot
					#   ignoring and treating the origin as lost
			${ECHO} "${N2_COLOR}      > origin DS base exists on ${remote_node_name}, but not the child dataset${N0_COLOR}"
			${ECHO} "${N2_COLOR}        from which our instance was created; ignoring; parent DS relation is lost.${N0_COLOR}"
		fi
	else
		IMGSND=0	# don't need to send over the origin DS
		${ECHO} "${N2_COLOR}      > origin DS exists on ${remote_node_name}, no need to import it${N0_COLOR}"
	fi
else
	IMGSND=3
	${ECHO} "${N2_COLOR}      > instance source origin does not reference ${jname}@final, relation to${N0_COLOR}"
	${ECHO} "${N2_COLOR}        dest. origin will be lost in the course of migrating the instance.${N0_COLOR}"
fi

if rexe tryoffline=1 node=${node} ". /etc/rc.conf && . \${cbsd_workdir}/cmd.subr && \${ZFS_CMD} list ${REMOTE_ZROOT}/${jname}@vmsnap-${CBSD_MIGRNAME}" > /dev/null 2>&1; then
	${ECHO} "${N1_COLOR}ERROR: Snapshot already exists on the ${remote_node_name}!  This is not${N0_COLOR}"
	${ECHO} "${N1_COLOR}       your first time here.  Clean it up first please.${N0_COLOR}"
	${ECHO} "${N1_COLOR}  To delete the snapshots on ${remote_node_name}:${N0_COLOR}"
	if [ "${emulator}" = "bhyve" ]; then
		${ECHO} "${N1_COLOR}    *${N2_COLOR}${SSH_CMD} ${remote_node_rip} \"${ZFS_CMD} destroy -r ${REMOTE_ZROOT}/${jname}/dsk0.vhd\"${N0_COLOR}"
		${ECHO} "${N1_COLOR}    *${N2_COLOR}${SSH_CMD} ${remote_node_rip} \"${ZFS_CMD} destroy -r ${REMOTE_ZROOT}/${jname}/dsk1.vhd\"${N0_COLOR}"
	fi

	${ECHO} "${N1_COLOR}    *${N2_COLOR}${SSH_CMD} ${remote_node_rip} \"${ZFS_CMD} destroy -r ${REMOTE_ZROOT}/${jname}\"${N0_COLOR}"
	clear_stfile
	exit 1
fi

#fsNotExist ${REMOTE_ZROOT}/${jname} && err 1 "${N1_COLOR}Error: ${node}, Dataset already exist: ${N2_COLOR}${REMOTE_ZROOT}/${jname}${N0_COLOR}"
#
#case "${emulator}" in
#	bhyve)
#		fsNotExist ${REMOTE_ZROOT}/${jname}/dsk0.vhd && err 1 "${N1_COLOR}Error: ${node}, Dataset already exist: ${N2_COLOR}${REMOTE_ZROOT}/${jname}/dsk0.vhd${N0_COLOR}"
#		fsNotExist ${REMOTE_ZROOT}/${jname}/dsk1.vhd && err 1 "${N1_COLOR}Error: ${node}, Dataset already exist: ${N2_COLOR}${REMOTE_ZROOT}/${jname}/dsk1.vhd${N0_COLOR}"
#	;;
#esac

if [ ${full} -eq 1 ]; then
	printf "${N1_COLOR}*${N0_COLOR} "
	jaillock="${jailsysdir}/${jname}/locked"
	[ -r "${jaillock}" ] && ${RM} ${jaillock}

	case "${emulator}" in
		jail)
			j2prepare node=${node} jname=${jname} mkdatadir=0 tryoffline=1
			ret=$?
			[ ${ret} -ne 0 ] && err 1 "${N1_COLOR}${CBSD_APP}: error: ${N2_COLOR}cbsd j2prepare node=${node} jname=${jname} mkdatadir=0 tryoffline=1${N0_COLOR}"
			;;
		bhyve)
			# export media data
			media mode=dump jname=${jname} > ${jailsysdir}/${jname}/media.sql
#			j2prepare node=${node} jname=${jname} mkdatadir=1 tryoffline=1

			# sharedfs = sync rcconf only
			j2prepare node=${node} jname=${jname} mkdatadir=0 tryoffline=1 sharedfs=1
			ret=$?
			[ ${ret} -ne 0 ] && err 1 "${N1_COLOR}${CBSD_APP}: error: ${N2_COLOR}cbsd j2prepare node=${node} jname=${jname} mkdatadir=1 tryoffline=1${N0_COLOR}"
#			j2slave node=${node} jname=${jname} tryoffline=1
#			ret=$?
#			[ ${ret} -ne 0 ] && err 1 "${N1_COLOR}${CBSD_APP}: error, no cbsdrsyncd service enabled on remote host?: ${N2_COLOR}j2slave node=${node} jname=${jname} tryoffline=1${N0_COLOR}"
			;;
	esac

	echo
fi

# Snapshotting and transfer
if [ ${incr} -eq 0 ]; then
	# NO INCREMENTAL MIGRATION
	DTIME=`${DATE_CMD} +%s`
	inst_stop
	# todo: remote modification
	${ECHO} "${N1_COLOR}* ${N2_COLOR}Creating dataset snapshots on ${my_nodename}${N0_COLOR}"
	for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
		${ECHO} "${N2_COLOR}    + Creating ${dset} snapshot...${N0_COLOR}"
		${ZFS_CMD} snapshot ${dset}@vmsnap-${CBSD_MIGRNAME} > /dev/null
	done

	echo

	${ECHO} "${N1_COLOR}* ${N2_COLOR}Transferring dataset snapshots to ${remote_node_name}${N0_COLOR}"
	if [ ${IMGSND} -eq 3 ]; then
		# origin DS ! exists on DEST, send over the instance breaking any parent DS relation
		for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
			${ECHO} "${N2_COLOR}    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME} ...${N0_COLOR}"
			echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
			_ret=$?

			DS_MNTPT=$( ${ZFS_CMD} get -Ho value mountpoint ${dset} )
			RDS_MNTPT=$( echo "${DS_MNTPT}" | ${SED_CMD} -e "s:${workdir}:${REMOTE_WORKDIR}:g" )

			echo "X [$dset], ${_ret}, MNTPT: [${RDS_MNTPT}]"

			if [ ${_ret} -eq 0 ]; then
				DNUM=$( ${BASENAME_CMD} "${dset}" )
				${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname}/${DNUM} mountpoint="${RDS_MNTPT}"
				_ret=$?
			else
				${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint="${RDS_MNTPT}"
				_ret=$?
			fi
			if [ ${_ret} -ne 0 ]; then
				${ECHO} "${N1_COLOR}      > 1transfer failed!${N0_COLOR}"
				clear_stfile
				exit 1
			fi
		done
	elif [ ${IMGSND} -eq 0 ]; then
		# origin DS exists on DEST, perform an incremental replication stream of instance dataset
		#   between the origin DS and the instance migration snapshot and send that to DEST
		for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
			${ECHO} "${N2_COLOR}    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME} ...${N0_COLOR}"
			if [ "${emulator}" = "bhyve" ]; then
				echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
				if [ $? -eq 0 ]; then
					# for KVMs, send the incremental replication stream only for dsk0
					#   since dsk0 is tied to the origin dataset, otherwise just send
					#   over the datasets
					#${SSH_CMD} -q -t ${remote_node_ip} "${ZFS_CMD} send -RI ${LOCAL_ZROOT}/${DSORIG} ${dset}@vmsnap-${CBSD_MIGRNAME} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT}" 2>/dev/null
					${SSH_CMD} -q -t ${remote_node_ip} "${ZFS_CMD} send -RI ${LOCAL_ZROOT}/${DSORIG} ${dset}@vmsnap-${CBSD_MIGRNAME} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT}/${jname}" 2>/dev/null
					if [ $? -ne 0 ]; then
						${ECHO} "${N1_COLOR}      > 2transfer failed!${N0_COLOR}"
						clear_stfile
						exit 1
					fi
				else
					${SSH_CMD} -q -t ${remote_node_ip} "${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT}" 2>/dev/null
					if [ $? -ne 0 ]; then
						${ECHO} "${N1_COLOR}      > 3transfer failed!${N0_COLOR}"
						clear_stfile
						exit 1
					fi
				fi
			else
				# for jails, only the root dataset exists, send an incremental replication stream
				transferAll ${dset}@vmsnap-${CBSD_MIGRNAME}
			fi
		done
	fi
else
	# INCREMENTAL MIGRATION
	USR1=0
	trap 'USR1=1' USR1
	BACKOFF=30
	while [ -f ${STFILE} ]; do
		${ECHO} "${N1_COLOR}* ${N2_COLOR}Creating dataset snapshots on ${my_nodename}${N0_COLOR}"
		for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
			${ECHO} "${N2_COLOR}    + Creating ${dset} snapshot ${cntIncr}...${N0_COLOR}"
			${ZFS_CMD} snapshot ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} > /dev/null
			echo "${dset}" | ${GREP_CMD} disk >/dev/null 2>&1
		done

		echo

		echo "* Transferring incremental dataset snapshot ${cntIncr} to ${remote_node_name} [IMGSND: ${IMGSND}]"
		if [ ${IMGSND} -eq 3 ]; then
			# origin DS ! exists on DEST, send over the instance breaking any parent DS relation
			if [ ${cntIncr} -eq 0 ]; then
				for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
					printf "    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} ..."
					DS_MNTPT=$( ${ZFS_CMD} get -Ho value mountpoint ${dset} )
					RDS_MNTPT=$( echo "${DS_MNTPT}" | ${SED_CMD} -e "s:${workdir}:${REMOTE_WORKDIR}:g" )
					echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
					if [ ${_ret} -eq 0 ]; then
						echo "${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint=\"${RDS_MNTPT}\" force=1"
						${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint="${RDS_MNTPT}" force=1
						_ret=$?
					else
						${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT} mountpoint="${RDS_MNTPT}" force=1
						_ret=$?
					fi

					if [ ${_ret} -ne 0 ]; then
						${ECHO} "${N1_COLOR}      > 4transfer failed!${N0_COLOR}"
						clear_stfile
						exit 1
					fi
				done
			else
				for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
					${ECHO} "    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} ...${N0_COLOR}"
					echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
					if [ ${_ret} -eq 0 ]; then
						${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint="${RDS_MNTPT}" force=1
						_ret=$?
					else
						${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT} mountpoint="${RDS_MNTPT}" force=1
						_ret=$?
					fi

					# old
					#${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -F ${dset} 2>/dev/null

					if [ ${_ret} -ne 0 ]; then
						${ECHO} "${N1_COLOR}      > 5transfer failed!${N0_COLOR}"
						clear_stfile
						exit 1
					fi
				done
			fi
		elif [ ${IMGSND} -eq 0 ]; then
			# origin DS exists on DEST, perform an incremental replication stream of instance dataset
			#   between the origin DS and the instance migration snapshot and send that to DEST
			if [ ${cntIncr} -eq 0 ]; then
				for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
					${ECHO} "${N2_COLOR}    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} ...${N0_COLOR}"
					if [ "${emulator}" = "bhyve" ]; then
						echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
						if [ $? -eq 0 ]; then
							# for KVMs, send the incremental replication stream only for dsk0
							#   since dsk0 is tied to the origin dataset, otherwise just send
							#   over the datasets
							#${ZFS_CMD} send -RI ${LOCAL_ZROOT}/${DSORIG} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT}/${jname} 2>/dev/null
							${ZFS_CMD} send -RI ${LOCAL_ZROOT}/${DSORIG} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint="${RDS_MNTPT}" force=1
							if [ $? -ne 0 ]; then
								${ECHO} "${N1_COLOR}      > 6transfer failed!${N0_COLOR}"
								clear_stfile
								exit 1
							fi
						else
							#${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -e ${REMOTE_ZROOT} 2>/dev/null
							${ZFS_CMD} send -p ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT} mountpoint="${RDS_MNTPT}" force=1
							if [ $? -ne 0 ]; then
								${ECHO} "${N1_COLOR}      > 7transfer failed!${N0_COLOR}"
								clear_stfile
								exit 1
							fi
						fi
					else
						# for jails, only the root dataset exists, send an incremental replication stream
						transferAll ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr}
					fi
				done
			else
				for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
					${ECHO} "${N2_COLOR}    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} ...${N0_COLOR}"
					#${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -F ${dset} 2>/dev/null
					echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
					if [ $? -eq 0 ]; then
						${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint="${RDS_MNTPT}" force=1
						_ret=$?
					else
						${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT} mountpoint="${RDS_MNTPT}" force=1
						_ret=$?
					fi
					if [ ${_ret} -ne 0 ]; then
						${ECHO} "${N1_COLOR}      > 8transfer failed!${N0_COLOR}"
						clear_stfile
						exit 1
					fi
				done
			fi
		fi

		cntPrev=${cntIncr}
		cntIncr=$(( cntIncr + 1 ))
		echo
		if [ ${aincr} -eq 0 ]; then
			${ECHO} "${N2_COLOR}    + Sleeping for ${BACKOFF} seconds before next snapshot (run 'kill -USR1 $$' to force a snapshot).${N0_COLOR}"
			for i in $( ${SEQ_CMD} 1 ${BACKOFF} ); do
				[ -f $STFILE ] || break
				if [ "$USR1" = 1 ]; then
					USR1=0
					break
				fi
				sleep 1
			done
			[ $BACKOFF -lt 480 ] && BACKOFF=$(( 2 * BACKOFF ))
		else
			clear_stfile
		fi
	done
	DTIME=`${DATE_CMD} +%s`
	inst_stop
	echo
	${ECHO} "${N1_COLOR}* ${N2_COLOR}Creating final incrmental dataset snapshot on ${my_nodename}${N0_COLOR}"
	for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
		${ECHO} "${N2_COLOR}    + Creating ${dset} snapshot ${cntIncr}...${N0_COLOR}"
		${ZFS_CMD} snapshot ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} > /dev/null
	done

	for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
		${ECHO} "${N2_COLOR}    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} ...${N0_COLOR}"
		#${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -F ${dset} 2>/dev/null

		DS_MNTPT=$( ${ZFS_CMD} get -Ho value mountpoint ${dset} )
		RDS_MNTPT=$( echo "${DS_MNTPT}" | ${SED_CMD} -e "s:${workdir}:${REMOTE_WORKDIR}:g" )
		echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
		_ret=$?
		echo "RET: ${_ret} ( dset: ${dset} )"
		if [ ${_ret} -eq 1 ]; then
			echo "${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint=\"${RDS_MNTPT}\" force=1"
			${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint="${RDS_MNTPT}" force=1
			_ret=$?
		else
			echo "${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT} mountpoint=\"${RDS_MNTPT}\" force=1"
			${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT} mountpoint="${RDS_MNTPT}" force=1
			_ret=$?
		fi
		if [ ${_ret} -ne 0 ]; then
			${ECHO} "${N1_COLOR}      > 9transfer failed!${N0_COLOR}"
			clear_stfile
			exit 1
		fi
	done


###XOO
#	for dset in `${AWK} '{printf $1}' ${tmpdir}/.vmmigrator.$$` ; do
#		${ECHO} "${N2_COLOR}    + Tranferring ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} ...${N0_COLOR}"
#		#${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | ${SSH_CMD} ${OPTSSH_CMD} ${remote_node_rip} ${ZFS_CMD} recv -F ${dset} 2>/dev/null
#		echo "${dset}" | ${GREP_CMD} disk >/dev/null 2>&1
#		_ret=$?
#		if [ ${_ret} -eq 0 ]; then
#			echo "${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint=\"${RDS_MNTPT}\" force=1"
#			${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT}/${jname} mountpoint="${RDS_MNTPT}" force=1
#			_ret=$?
#		else
#			${ZFS_CMD} send -i ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntPrev} ${dset}@vmsnap-${CBSD_MIGRNAME}-${cntIncr} | cbsd rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata=${REMOTE_ZROOT} mountpoint="${RDS_MNTPT}" force=1
#			_ret=$?
#		fi
#		if [ ${_ret} -ne 0 ]; then
#			${ECHO} "${N1_COLOR}      > 9transfer failed!${N0_COLOR}"
#			clear_stfile
#			exit 1
#		fi
#	done
fi

${ECHO} "${N1_COLOR}* ${N2_COLOR}Deleting migration snapshots on ${remote_node_name}...${N0_COLOR}"
# Also necessary remove on local, because at the end of script, user can anser 'no' on 'remove vm?' question
for dset in $( ${AWK_CMD} '{printf $1"\n"}' ${tmpdir}/.vmmigrator.$$ ); do
	echo "${dset}" | ${GREP_CMD} -E "dsk[0-9]+.vhd" >/dev/null 2>&1
	_ret=$?
	echo "RET: ${_ret} ( DESTROY dset: ${dset} )"


	if [ ${incr} -eq 0 ]; then
		if [ ${_ret} -eq 0 ]; then
			target=$( ${BASENAME_CMD} "${dset}" )
			DESTROY_SNAP="${REMOTE_ZROOT}/${jname}/${target}@vmsnap-${CBSD_MIGRNAME}"
		else
			DESTROY_SNAP="${REMOTE_ZROOT}/${jname}@vmsnap-${CBSD_MIGRNAME}"
		fi
		echo "destroy snapshot on remote host...: [${ZFS_CMD} destroy ${DESTROY_SNAP}]"
		rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata="${DESTROY_SNAP}" mountpoint="destroy"
		#rexe node=zfsmigrat2.my.domain tryoffline=1 /usr/local/bin/cbsd zfs-recv tank/jails2 ${SSH_CMD} -q ${remote_node_rip} "${ZFS_CMD} destroy ${dset}@vmsnap-${CBSD_MIGRNAME}" 2>/dev/null
		${ZFS_CMD} destroy ${dset}@vmsnap-${CBSD_MIGRNAME} 2>/dev/null
	else
		prevCnt=${cntIncr}
		while [ ${prevCnt} -ge 0 ]; do
			#${SSH_CMD} -q ${remote_node_rip} "${ZFS_CMD} destroy ${dset}@vmsnap-${CBSD_MIGRNAME}-${prevCnt}" 2>/dev/null

			if [ ${_ret} -eq 0 ]; then
				target=$( ${BASENAME_CMD} "${dset}" )
				DESTROY_SNAP="${REMOTE_ZROOT}/${jname}/${target}@vmsnap-${CBSD_MIGRNAME}-${prevCnt}"
			else
				DESTROY_SNAP="${REMOTE_ZROOT}/${jname}@vmsnap-${CBSD_MIGRNAME}-${prevCnt}"
			fi

			rexe node=${node} tryoffline=1 /usr/local/bin/cbsd zfs-recv jname=${jname} zdata="${DESTROY_SNAP}" mountpoint="destroy"
			${ZFS_CMD} destroy ${dset}@vmsnap-${CBSD_MIGRNAME}-${prevCnt} 2>/dev/null
			#let prevCnt=prevCnt-1
			prevCnt=$(( prevCnt - 1 ))
		done
	fi
done

${RM} -f ${tmpdir}/.vmmigrator.$$

# Disable VM on SRC
${ECHO} "${N1_COLOR}* ${N2_COLOR}Disabling instance on ${my_nodename}...${N0_COLOR}"

case "${emulator}" in
	jail)
		jset jname=${jname} astart=0 > /dev/null 2>&1
		;;
	bhyve)
		bset jname=${jname} astart=0 > /dev/null 2>&1
		;;
esac

${ECHO} "${N1_COLOR}* ${N2_COLOR}Register instance on ${remote_node_name}...${N0_COLOR}"

if [ ${full} -eq 1 ]; then

	rexe node=${node} tryoffline=1 /usr/local/bin/cbsd jregister jname=${jname}

	if [ "${jail_status}" = "running" ]; then
		${ECHO} "${N1_COLOR}* ${N2_COLOR}Run jail on ${node}${N0_COLOR}"
		case "${emulator}" in
			jail)
				rexe node=${node} tryoffline=1 /usr/local/bin/cbsd jstart jname=${jname} inter=0
				;;
			bhyve)
				rexe node=${node} tryoffline=1 /usr/local/bin/cbsd bstart jname=${jname} inter=0
				;;
		esac
	fi
fi

ETIME=`${DATE_CMD} +%s`
ENDTIME=`${DATE_CMD}`

clear_stfile
echo ""
printf "\n                    ===   Done! ===\n"
printf "\n   >> ${emulator} instance:  ${jname} (${host_hostname})"
printf "\n          is now installed on\n"
echo "   >> dest node:  ${remote_node_name} (${dest_vm_name})"
echo ""
echo "    Migration started:  ${BEGTIME}"
echo "      Migration ended:  ${ENDTIME}"

TDIFF=`expr ${ETIME} - ${STTIME}`
if [ ${TDIFF} -gt 3600 ]; then
	TDIFF=`echo "scale=2; ${TDIFF}/3600" | ${BC}`
	TDIFF="${TDIFF} hours"
elif [ ${TDIFF} -gt 60 ]; then
	TDIFF=`echo "scale=2; ${TDIFF}/60" | ${BC}`
	TDIFF="${TDIFF} minutes"
else
	TDIFF="${TDIFF} seconds"
fi
DIFFT=`expr ${ETIME} - ${DTIME}`
if [ ${DIFFT} -gt 3600 ]; then
	DIFFT=`echo "scale=2; ${DIFFT}/3600" | ${BC}`
	DIFFT="${DIFFT} hours"
elif [ ${DIFFT} -gt 60 ]; then
	DIFFT=`echo "scale=2; ${DIFFT}/60" | ${BC}`
	DIFFT="${DIFFT} minutes"
else
	DIFFT="${DIFFT} seconds"
fi

echo "Duration of migration:  ${TDIFF}"
echo "    Instance downtime:  ${DIFFT}"
echo "       Migration type:  ${migrType}"

cntIncr=$(( cntIncr + 1 ))
echo "   Dataset increments:  ${cntIncr}"
if [ ${incr} -gt 0 ]; then
	echo "   migration log file:  ${DC_NAME}:${LOGFILE}"
fi

#
# Migration Summary - Originating node
#

echo "          Source node:  ${my_nodename} (${source_vm_name})"
echo

if [ ${aremove} -eq 0 -o ${aremove} -eq 1 ]; then
	# store original ALWAYS_ and inter settings
	OALWAYS_YES=${ALWAYS_YES}
	OALWAYS_NO=${ALWAYS_NO}
	ointer=${inter}
	inter=0
	unset ALWAYS_YES
	unset ALWAYS_NO

	case ${aremove} in
		0)
			ALWAYS_NO=1
			;;
		1)
			ALWAYS_YES=1
			;;
	esac
fi

# Ask user if they would like to cleanup now...
if getyesno "Would you like to remove this instance from ${my_nodename} now? "; then
	if [ "${emulator}" = "bhyve" ]; then
		bremove jname=${jname}
		#echo "* ${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}/dsk1.vhd"
		#(${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}/dsk1.vhd && echo "  ...success") || echo "  ...FAILED."
		#echo "* ${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}/dsk2.vhd\""
		#(${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}/dsk2.vhd && echo "  ...success") || echo "  ...FAILED."
	else
		jremove jname=${jname}
		#echo "* ${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}"
		#(${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname} && echo "  ...success") || echo "  ...FAILED."
	fi
else
	echo "    The source node should be cleaned of this VM.              "
	echo ""
	echo "    - To finish removing the VM from the source (${my_nodename}):      "
	echo ""
	if [ "${emulator}" = "bhyve" ]; then
		echo "      bremove jname=${jname}"
		echo "    or"
		echo "      ${SSH_CMD} ${remote_node_ip} \"${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}/dsk0.vhd\" &&"
		echo "      ${SSH_CMD} ${remote_node_ip} \"${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}/dsk1.vhd\" &&"
	else
		echo "      jremove jname=${jname}"
		echo "    or"
		echo "      ${ZFS_CMD} destroy -r ${LOCAL_ZROOT}/${jname}"
	fi
	echo
fi

# restore original settings
if [ ${aremove} -eq 0 -o ${aremove} -eq 1 ]; then
	# store original ALWAYS_ and inter settings
	ALWAYS_YES=${OALWAYS_YES}
	ALWAYS_NO=${OALWAYS_NO}
	inter=${ointer}
fi

echo
exit 0
