#!/usr/local/bin/cbsd
#v11.2.0
MYARG="jname mode"
MYOPTARG="name suspend"
MYDESC="bhyve checkpoint"
CBSDMODULE="bhyve"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

 The functionality of checkpoints and suspend of a virtual machine is the
 freezing of the virtual environment, saving the entire state to disk,
 from which you can return the system to its previous state without
 having to reboot the environment. CBSD bcheckpoint script helps you 
 to manage and organize checkints.

 Do not confuse this feature with disk snapshots (cbsd bsnapshot),
 although a combination of 'cbsd checkpoint suspend=1' + 'cbsd bsnapshot'
 can be used together. Make sure that the checkpoint is always consistent 
 with the state of the disk at the time of the checkpoint.

 To run VM from a checkpoint, use 'cbsd bstart' with the 'checkpoint' argument.

 ${W1_COLOR}Warning! ${N0_COLOR}To use this feature, kernel (vmm.ko) and bhyve userland
 utilities (bhyve,bhyvectl) must be compiled with the BHYVE_SNAPSHOT support:

  1) rebuild world/kernel with bhyve snapshot support, e.g. via: /etc/src.conf:
  WITH_BHYVE_SNAPSHOT=yes

  and kernel config options:

  options         BHYVE_SNAPSHOT

 Then rebuild the kernel/world.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}jname=${N0_COLOR}    - target envinvironment.
 ${N2_COLOR}mode=${N0_COLOR}     - action: create,list,destroyall:
   create     - create new checkpoint;
   list       - show available checkpoint for current domain;
   destroyall - destroy all checkpoints for current domain;
 ${N2_COLOR}name=${N0_COLOR}     - name of checkpoint. by default: 'checkpoint'.
 ${N2_COLOR}suspend=${N0_COLOR}  - when set to 1 then turn off the domain immediately after checkpoint,
             for disk consistency. By default - 0, create checkpoint only.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd bcheckpoint mode=create suspend=1 jname=myvm
 # cbsd bcheckpoint mode=list jname=myvm
 # cbsd bstart checkpoint=checkpoint jname=myvm

${H3_COLOR}See also${N0_COLOR}:

  cbsd bsnapshot --help
  cbsd bstart --help
  cbsd bpause --help

"

. ${subrdir}/nc.subr

suspend=0
name="checkpoint"

. ${cbsdinit}

# store original name before rcconf init
oname="${name}"

. ${subrdir}/rcconf.subr
[ $? -eq 1 ] && err 1 "${N1_COLOR}No such domains: ${N2_COLOR}${jname}${N0_COLOR}"
[ "${emulator}" != "bhyve" ] && log_err 1 "${N1_COLOR}Not in bhyve mode${N0_COLOR}"

# restore original name
name="${oname}"

CHECKPOINT_DIR="${jailsysdir}/${jname}/checkpoints"
[ ! -d ${CHECKPOINT_DIR} ] && ${MKDIR_CMD} -p ${CHECKPOINT_DIR}

checkpoint_create()
{
	[ ${jid} -eq 0 ] && log_err 1 "Not running"
	[ -z "${name}" ] && log_err 1 "${N1_COLOR}Empty checkpoint name${N0_COLOR}"

	CHECKPOINT="${CHECKPOINT_DIR}/${name}.ckp"

	# todo: check for .ckp extenions

	if [ -r ${CHECKPOINT} ]; then
		${ECHO} "${N1_COLOR}Prune old checkpoint...${N0_COLOR}"
		cbsdlogger NOTICE ${CBSD_APP}: prune old checkpoint: ${CHECKPOINT}
		${RM_CMD} -f ${CHECKPOINT} ${CHECKPOINT}.kern ${CHECKPOINT}.meta
	fi

	if [ ${suspend} -eq 1 ]; then
		# we must shutdown VM after the checkpoint is created.
		# We guarantee that it will not work any more, freezing its condition before snapshot.
		# In this life, VM will never wake up and we can kill VM hard
		# suspend -> crash for some reason. Use checkpoint instead
		${BHYVECTL_CMD} --suspend=${CHECKPOINT} --vm=${jname}
		#${BHYVECTL_CMD} --checkpoint=${CHECKPOINT} --vm=${jname}

		# if we kill or pause VM too early, the <chk>.kern file will not be generated
		# (why does control come back before the end, bhyve bug?)
		#bpause mode=on jname=${jname} || true
		# switch to RO?
		for i in $( ${SEQ_CMD} 1 2 ); do
			${ECHO} "${N1_COLOR}Waiting and sure that the info is written on the disk: ${N2_COLOR}${i}/5${N0_COLOR}"
			sleep 1
		done
		# kill the virtual machine immediately
		sync
		bstop jname=${jname} noacpi=1 || true
		cbsdlogger NOTICE ${CBSD_APP}: stop ${jname} due to checkpoint suspend=1: ${CHECKPOINT}
	else
		# switch to dsk RO ?
		${BHYVECTL_CMD} --checkpoint ${CHECKPOINT} --vm=${jname}
	fi

	# todo:
	# check for <chk>{.kern,meta} size > 0
	# check for valid meta json

	sync
	${ECHO} "${N1_COLOR}checkpoint was created: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
	return 0
}

checkpoint_list()
{
	${ECHO} "${N1_COLOR}Created checkpoint for ${N2_COLOR}${jname}${N1_COLOR}:${N0_COLOR}"
	${FIND_CMD} ${CHECKPOINT_DIR}/  -mindepth 1 -maxdepth 1 -name \*.ckp -type f -exec ${BASENAME_CMD} {} \; | ${SORT_CMD} | while read _file; do
		p1=${_file%%.*}
		p2=${_file##*.}
		echo ${p1}
	done
}

case "${mode}" in
	create)
		checkpoint_create
		;;
	destroyall)
		[ -d ${CHECKPOINT_DIR} ] && ${RM_CMD} -rf ${CHECKPOINT_DIR}
		;;
	list)
		checkpoint_list
		;;
	*)
		err 1 "${N1_COLOR}Unknown mode: ${N2_COLOR}${mode}${N0_COLOR}"
esac

exit 0
