#!/usr/local/bin/cbsd
#v12.1.10
MYARG="mode"
MYOPTARG="bridge_ips display dryrun force header vpc_name peer_network node_member"
MYDESC="Operate with CBSD VPC: vxlan-based multi-node full-mesh network overlay"
CBSDMODULE="sys"
ADDHELP="\

${H3_COLOR}Description${N0_COLOR}:

'cbsd vpc' script helps to create an overlay (L2) network between CBSD nodes based on vxlan technology.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}bridge_ips${N0_COLOR}            - (for mode=init_bridge) add ips address (comma separated) to vpc bridge.
 ${N2_COLOR}display=<item1,item2>${N0_COLOR} - list by comma for column. Default: name,pkg_vm_cpus,pkg_vm_ram,pkg_vm_disk.
 ${N2_COLOR}dryrun=1${N0_COLOR}              - print command only, dont apply.
 ${N2_COLOR}force=1${N0_COLOR}               - Form of action locally, even remote node is not available.
 ${N2_COLOR}header=0${N0_COLOR}              - don't print header.
 ${N2_COLOR}mode=${N0_COLOR}                 - init,remove,list,init_peers,init_vxlan,destroy_vxlan,sync,
                         init_bridge,destroy_bridge,deploy,destroy,status:
  mode=init           - init/create VPC (locally);
  mode=init_peers     - generate peer map (locally);
  mode=init_vxlan     - apply peer map, create vxlan cfg (locally);
  mode=destroy_vxlan  - destroy peer map, remove vxlan cfg (locally);
  mode=init_bridge    - apply VPC bridge, create bridge cfg (locally);
  mode=destroy_bridge - destroy VPC map, remove bridge cfg (locally);
  mode=deploy         - mass initialization on all nodes (init_peers + init_vxlan + init_bridge);
  mode=collapse       - mass de-initialization on all nodes (destroy_vxlan + destroy_bridge);
  mode=status         - get VPC state;
  mode=init_rc        - create FreeBSD rc.d startup script for init VPC upon host boot;
 ${N2_COLOR}node_member${N0_COLOR}           - (for mode=create) can be 'all'
 ${N2_COLOR}peer_networkember${N0_COLOR}     - system/endpoint network for full-mesh point-to-point

${H3_COLOR}Examples${N0_COLOR}:

 (when node member added via 'cbsd node mode=add')

   host1:
     a) cbsd vpc mode=init node_member=all peer_network=10.100.0.0/24 vpc_name=vpc1
   host2:
     a) cbsd vpc mode=init node_member=all peer_network=10.100.0.0/24 vpc_name=vpc1
   host1:
     b) cbsd vpc vpc_name=vpc1 mode=init_peers
     c) cbsd vpc vpc_name=vpc1 mode=sync
   all host:
     d) cbsd vpc mode=list
     e) cbsd vpc vpc_name=vpc1 mode=init_vxlan
     f) cbsd vpc vpc_name=vpc1 mode=init_bridge [bridge_ips=X.X.X.X/24]
     g) cbsd vpc vpc_name=vpc1 mode=init_rc

"
EXTHELP="wf_vpc"

. ${subrdir}/nc.subr
dryrun=0
force=0
. ${cbsdinit}
. ${system}
. ${strings}

VPC_ROOT_DIR="${dbdir}/vpc"

[ ! -d ${VPC_ROOT_DIR} ] && ${MKDIR_CMD} ${VPC_ROOT_DIR}

get_vxlan_ip()
{
	local _id="${1}"
	local _ip

	eval _ip="\$HOST${_id}_VXLAN_IP"
	[ -z "${_ip}" ] && err 1 "Unable to determine remote VXLAN for node id $i"
	printf "${_ip}"
}

init_vpc()
{
	local _dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	local _nodes _node_ip _all_nodes _res

	_all_nodes=$( cbsdsqlro nodes "SELECT nodename FROM nodelist" | ${XARGS_CMD} | ${TR_CMD} " " "," )

	if [ -z "${_all_nodes}" ]; then
		err 1 "${N1_COLOR}no any node found in cluster, please use first: ${N2_COLOR}cbsd node mode=add${N0_COLOR}"
	fi

	if [ -r ${_dbpath} ]; then
		_res=$( cbsdsqlro ${_dbpath} "SELECT init_node FROM vpc LIMIT 1" )
		err 1 "${N1_COLOR}VPC already initialized by: ${N2_COLOR}${_res} ${N1_COLOR}( ${_dbpath} )${N0_COLOR}"
	fi

	if [ "${node_member}" = "all" ]; then
		# special case to get all nodes
		node_member="${_all_nodes}"
	else
		OIFS="${IFS}"
		IFS=","

		for i in ${node_member}; do
			IFS="${OIFS}"
			_node_ip=$( cbsdsqlro nodes "SELECT ip FROM nodelist WHERE nodename=\"${i}\" LIMIT 1" )
			if [ -z "${_node_ip}" ]; then
				${ECHO} "${N1_COLOR}no such node: ${N2_COLOR}${i}${N0_COLOR}"
				err 1 "${N1_COLOR}available nodes: ${N2_COLOR}${_all_nodes}${N0_COLOR}"
			fi
			IFS=","
		done
		IFS="${OIFS}"
	fi

	/usr/local/bin/cbsd ${miscdir}/updatesql ${_dbpath} ${distdir}/share/local-vpc.schema vpc

	cbsdsqlrw ${_dbpath} "INSERT INTO vpc ( init_node,vpc_name,node_member,peer_network ) VALUES ( \"${nodename}\", \"${vpc_name}\",\"${node_member}\",\"${peer_network}\" )"
	return 0
}

list_vpc()
{
	local _dbpath _list

	if [ -n "${vpc_name}" ]; then
		_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
		[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"
		cbsdsqlro ${_dbpath} "SELECT * FROM vpc"
	else
		_list=$( ${LS_CMD} -1 ${VPC_ROOT_DIR} | ${SED_CMD} 's:.sqlite::g' | ${GREP_CMD} -E -v "\-wal|\-shm" )
		[ -z "${_list}" ] && return 0

		vms=$( cbsdsqlro local "SELECT jname FROM jails WHERE emulator = \"bhyve\"" )
		jails=$( cbsdsqlro local "SELECT jname FROM jails WHERE emulator = \"jail\"" )

		vpc_members=

		for i in ${_list}; do
			for j in ${jails}; do
				mydb="${jailsysdir}/${j}/local.sqlite"
				[ ! -r "${mydb}" ] && continue
				_test=$( cbsdsqlro ${mydb} "SELECT name FROM jailnic WHERE nic_parent=\"vpc-${i}\" LIMIT 1" )
				if [ -n "${_test}" ]; then
					if [ -n "${vpc_members}" ]; then
						vpc_members="${vpc_members} ${j}"
					else
						vpc_members="${j}"
					fi
				fi
			done

			echo ":: ${i} [${vpc_members}]"
			vpc mode=list vpc_name="${i}"
		done
	fi
}

# populate vpc_peers table with peer map
init_peers()
{
	local _node_list _peer_network _res
	local _vxlan_st=1		# to config

	local _vxlan_id="${_vxlan_st}"

	_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"
	_node_list=$( cbsdsqlro ${_dbpath} SELECT node_member FROM vpc LIMIT 1 | ${AWK_CMD} '{printf $1}' | ${TR_CMD} ',' ' ' )
	_peer_network=$( cbsdsqlro ${_dbpath} SELECT peer_network FROM vpc LIMIT 1 | ${AWK_CMD} '{printf $1}' | ${TR_CMD} ',' ' ' )

	_node_list="${nodename} ${_node_list}"

	# check nodes in addresses_in_network
	echo "nodes: ${_node_list}" 1>&2
	echo "peer_network: ${_peer_network}" 1>&2

	eval $( ${miscdir}/sipcalc ${_peer_network} )
	[ -z "${_addresses_in_network}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine addresses in network: ${_peer_network}${N0_COLOR}"
	[ -z "${_usable_range_start}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine usable range start: ${_peer_network}${N0_COLOR}"
	[ -z "${_usable_range_end}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine usable range end: ${_peer_network}${N0_COLOR}"

	sqllistdelimer="."
	sqllist "${_usable_range_start}" _s1 _s2 _s3 _s4
	cbsdsqlrw ${_dbpath} "DROP TABLE IF EXISTS vpc_peers"		# delete all prev records
	/usr/local/bin/cbsd ${miscdir}/updatesql ${_dbpath} ${distdir}/share/local-vpc-peers.schema vpc_peers

	for node_st in ${_node_list}; do
		for node_dst in ${_node_list}; do
			[ "${node_st}" = "${node_dst}" ] && continue	# skip to myself

			# test for dup first
			_res=$( cbsdsqlro ${_dbpath} "SELECT id FROM vpc_peers WHERE src_node=\"${node_st}\" AND dst_node=\"${node_dst}\" LIMIT 1" | ${AWK_CMD} '{printf $1}' )

			if [ -z "${_res}" ]; then
				_ip="${_s1}.${_s2}.${_s3}.${_s4}/30"
				echo "${node_st} -> ${node_dst} , ep: ${_ip}  (VX ID: ${_vxlan_id})" 1>&2
				cbsdsqlrw ${_dbpath} "INSERT INTO vpc_peers ( src_node,dst_node,peer_ip,vxlan_id ) VALUES ( \"${node_st}\",\"${node_dst}\",\"${_ip}\",\"${_vxlan_id}\" )"

				_s4=$(( _s4 + 1 ))
				_ip="${_s1}.${_s2}.${_s3}.${_s4}/30"
				echo "  ${node_dst} -> ${node_st}, ep: ${_ip}  (VX ID: ${_vxlan_id})" 1>&2
				cbsdsqlrw ${_dbpath} "INSERT INTO vpc_peers ( src_node,dst_node,peer_ip,vxlan_id ) VALUES ( \"${node_dst}\",\"${node_st}\",\"${_ip}\",\"${_vxlan_id}\" )"
				_s4=$(( _s4 + 3 ))
				_vxlan_id=$(( _vxlan_id + 1 ))
			fi
		done
	done
}

# ifconfig vxlan23 create vxlanid 23 vxlanlocal 2a05:3580:d800:20f7::1 vxlanremote 2a01:4f8:241:500b::1 inet 10.10.23.3/24 mtu 1450 up
init_vxlan()
{
	local _node_list _peer_network _myip= _dbpath _res _ret

	_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	_peer_network=$( cbsdsqlro ${_dbpath} "SELECT dst_node FROM vpc_peers LIMIT 1" )
	[ -z "${_peer_network}" ] && err 1 "${N1_COLOR}peer not initialized, please use first: ${N2_COLOR}cbsd vpc mode=init_peers vpc_name=${vpc_name}${N0_COLOR}"

	cbsdsqlro ${_dbpath} "SELECT dst_node,peer_ip,vxlan_id FROM vpc_peers WHERE src_node=\"${nodename}\"" | while read _line; do
		sqllist "${_line}" dst_node peer_ip vxlan_id
		_node_ip=$( cbsdsqlro nodes SELECT ip FROM nodelist WHERE nodename=\"${dst_node}\" )
		[ -z "${_node_ip}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine ip for node: ${N2_COLOR}${dst_node}${N0_COLOR}"
		iptype ${_node_ip} > /dev/null 2>&1
		case $? in
			0)
				err 1 "${N1_COLOR}${CBSD_APP}: unable to determine ip type for: ${N2_COLOR}${_node_ip}${N0_COLOR}"
				;;
			1)
				iptype ${nodeip} > /dev/null 2>&1
				case $? in
					0)
						${ECHO} "${N1_COLOR}${CBSD_APP}: unable to determine ip type for nodeip: ${N2_COLOR}${nodeip}${N0_COLOR}"
						err 1 "${N1_COLOR}${CBSD_APP}: please set it via: ${N2_COLOR}cbsd initenv-tui${N0_COLOR}"
						;;
					*)
						;;
				esac

				_myip="${nodeip}"
				;;
			2)
				iptype ${nodeip6} > /dev/null 2>&1
				case $? in
					0)
						${ECHO} "${N1_COLOR}${CBSD_APP}: unable to determine ip type for nodeip6: ${N2_COLOR}${nodeip6}${N0_COLOR}"
						err 1 "${N1_COLOR}${CBSD_APP}: please set it via: ${N2_COLOR}cbsd initenv-tui${N0_COLOR}"
						;;
					*)
						;;
				esac
				_myip="${nodeip6}"
				;;
		esac
		${IFCONFIG_CMD} vxlan${vxlan_id} up > /dev/null 2>&1
		_ret=$?
		if [ ${_ret} -eq 0 ]; then
			${ECHO} "${N1_COLOR}interface already exist: ${N2_COLOR}vxlan${vxlan_id}. ${N1_COLOR}Use for destroy: ${N2_COLOR}cbsd vpc mode=destroy_vxlan vpc_name=${vpc_name}${N0_COLOR}"
			continue
		fi

		echo "${IFCONFIG_CMD} vxlan${vxlan_id} create vxlanid ${vxlan_id} vxlanlocal ${_myip} vxlanremote ${_node_ip} inet ${peer_ip} mtu 1450 up"
		_res="${IFCONFIG_CMD} vxlan${vxlan_id} create vxlanid ${vxlan_id} vxlanlocal ${_myip} vxlanremote ${_node_ip} inet ${peer_ip} mtu 1450 up"
		if [ ${dryrun} -eq 1 ]; then
			echo "${_res}"
			_ret=$?
		else
			${_res}
			_ret=$?
		fi
		if [ ${_ret} -ne 0 ]; then
			err 0 "${N1_COLOR}vxlan init failed: ${N2_COLOR}${_res}${N0_COLOR}"
		fi
		${IFCONFIG_CMD} vxlan${vxlan_id} description tunnel-to-${dst_node} > /dev/null 2>&1
		${IFCONFIG_CMD} vxlan${vxlan_id} down
		${IFCONFIG_CMD} vxlan${vxlan_id} up
	done
}

destroy_vxlan()
{
	local _node_list _peer_network _myip= _dbpath _res _ret

	_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	_peer_network=$( cbsdsqlro ${_dbpath} "SELECT dst_node FROM vpc_peers LIMIT 1" )
	[ -z "${_peer_network}" ] && err 1 "${N1_COLOR}peer not initialized, please use first: ${N2_COLOR}cbsd vpc mode=init_peers vpc_name=${vpc_name}${N0_COLOR}"

	cbsdsqlro ${_dbpath} "SELECT dst_node,peer_ip,vxlan_id FROM vpc_peers WHERE src_node=\"${nodename}\"" | while read _line; do
		sqllist "${_line}" dst_node peer_ip vxlan_id
		_node_ip=$( cbsdsqlro nodes SELECT ip FROM nodelist WHERE nodename=\"${dst_node}\" )
		[ -z "${_node_ip}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine ip for node: ${N2_COLOR}${dst_node}${N0_COLOR}"
		iptype ${_node_ip} > /dev/null 2>&1
		case $? in
			0)
				[ -z "${_node_ip}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine ip type for: ${N2_COLOR}${_node_ip}${N0_COLOR}"
				;;
			1)
				_myip="${nodeip}"
				;;
			2)
				_myip="${nodeip6}"
				;;
		esac
		${IFCONFIG_CMD} vxlan${vxlan_id} > /dev/null 2>&1
		_ret=$?
		if [ ${_ret} -ne 0 ]; then
			${ECHO} "${N1_COLOR}interface not exist: ${N2_COLOR}vxlan${vxlan_id}${N0_COLOR}"
			continue
		fi
		_res="${IFCONFIG_CMD} vxlan${vxlan_id} destroy"
		if [ ${dryrun} -eq 1 ]; then
			echo "${_res}"
			_ret=$?
		else
			${_res}
			_ret=$?
		fi
		_ret=$?
		if [ ${_ret} -ne 0 ]; then
			err 0 "${N1_COLOR}vxlan destroy failed: ${N2_COLOR}${_res}${N0_COLOR}"
		fi
	done
}

init_bridge()
{
	local _node_list _peer_network _myip= _dbpath _res _ret _bridge_id

	_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	_peer_network=$( cbsdsqlro ${_dbpath} "SELECT dst_node FROM vpc_peers LIMIT 1" )
	[ -z "${_peer_network}" ] && err 1 "${N1_COLOR}peer not initialized, please use first: ${N2_COLOR}cbsd vpc mode=init_peers vpc_name=${vpc_name}${N0_COLOR}"

	cbsdsqlro ${_dbpath} "SELECT dst_node,peer_ip,vxlan_id FROM vpc_peers WHERE src_node=\"${nodename}\"" | while read _line; do
		sqllist "${_line}" dst_node peer_ip vxlan_id
		_node_ip=$( cbsdsqlro nodes SELECT ip FROM nodelist WHERE nodename=\"${dst_node}\" )
		[ -z "${_node_ip}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine ip for node: ${N2_COLOR}${dst_node}${N0_COLOR}"
		iptype ${_node_ip} > /dev/null 2>&1
		case $? in
			0)
				[ -z "${_node_ip}" ] && err 1 "${N1_COLOR}${CBSD_APP}: unable to determine ip type for: ${N2_COLOR}${_node_ip}${N0_COLOR}"
				;;
			1)
				iptype ${nodeip} > /dev/null 2>&1
				case $? in
					0)
						${ECHO} "${N1_COLOR}${CBSD_APP}: unable to determine ip type for nodeip: ${N2_COLOR}${nodeip}${N0_COLOR}"
						err 1 "${N1_COLOR}${CBSD_APP}: please set it via: ${N2_COLOR}cbsd initenv-tui${N0_COLOR}"
						;;
					*)
						;;
				esac
				_myip="${nodeip}"
				;;
			2)
				case $? in
					0)
						${ECHO} "${N1_COLOR}${CBSD_APP}: unable to determine ip type for nodeip6: ${N2_COLOR}${nodeip6}${N0_COLOR}"
						err 1 "${N1_COLOR}${CBSD_APP}: please set it via: ${N2_COLOR}cbsd initenv-tui${N0_COLOR}"
						;;
					*)
						;;
				esac
				_myip="${nodeip6}"
				;;
		esac
		${IFCONFIG_CMD} vxlan${vxlan_id} up > /dev/null 2>&1
		_ret=$?
		[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}interface not exist: ${N2_COLOR}vxlan${vxlan_id}${N1_COLOR}. please init first: ${N1_COLOR}cbsd vpc mode=init_vxlan vpc_name=${vpc_name}${N0_COLOR}"
	done

#	_bridge_id=$( cbsdsqlro ${_dbpath} "SELECT dst_node FROM vpc_peers LIMIT 1" )
	if [ -z "${bridge_ips}" -o "${bridge_ips}" = "0" ]; then
		bridge_ips=$( cbsdsqlro ${_dbpath} "SELECT bridge_ips FROM vpc" | ${AWK_CMD} '{printf $1}' )
	fi

	. ${subrdir}/vnet.subr

	mybridge=
	if ! mybridge=$( get_my_device vpc vpc-${vpc_name} ); then
		${ECHO} "${N1_COLOR}${CBSD_APP} failed to get bridge: ${mybridge}${N0_COLOR}"
		return 1
	fi

	if [ -n "${bridge_ips}" -a "${bridge_ips}" != "0" ]; then
		# todo: comma + iptype
		${ECHO} "${N1_COLOR}add ips for vpc bridge: ${N2_COLOR}${IFCONFIG_CMD} ${mybridge} inet ${bridge_ips} up${N0_COLOR}"
		${IFCONFIG_CMD} ${mybridge} inet ${bridge_ips}
		${IFCONFIG_CMD} ${mybridge} down
		${IFCONFIG_CMD} ${mybridge} up
		cbsdsqlrw ${_dbpath} "UPDATE vpc SET bridge_ips=\"${bridge_ips}\""
	fi

	# refresh vxlan iface again (bug upon start?)
	cbsdsqlro ${_dbpath} "SELECT vxlan_id FROM vpc_peers WHERE src_node=\"${nodename}\"" | while read _line; do
		sqllist "${_line}" vxlan_id
		${IFCONFIG_CMD} vxlan${vxlan_id} down
		${IFCONFIG_CMD} vxlan${vxlan_id} up
	done

	${ECHO} "${N1_COLOR}vpc bridge created: ${N2_COLOR}${mybridge}${N0_COLOR}"
	${ECHO} "${N1_COLOR}you may want to create startup script via: ${N2_COLOR}cbsd vpc vpc_name=${vpc_name} mode=init_rc${N0_COLOR}"
}

init_rc()
{
	local _node_list _peer_network _myip= _dbpath _res _ret _bridge_id

	_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	[ ! -r ${distsharedir}/vpc-rc.tpl ] && err 1 "${N1_COLOR}${CBSD_APP}: no such ${distsharedir}/vpc-rc.tpl template${N0_COLOR}"

	if [ -z "${bridge_ips}" -o "${bridge_ips}" = "0" ]; then
		bridge_ips=$( cbsdsqlro ${_dbpath} "SELECT bridge_ips FROM vpc" | ${AWK_CMD} '{printf $1}' )
	fi

	. ${subrdir}/vnet.subr

	mybridge=
	if ! mybridge=$( get_my_device vpc vpc-${vpc_name} ); then
		${ECHO} "${N1_COLOR}${CBSD_APP} failed to get bridge: ${mybridge}${N0_COLOR}"
		return 1
	fi

	${SED_CMD} -e "s:%%VPC_NAME%%:${vpc_name}:g" \
		-e "s:%%BRIDGE_IPS%%:${bridge_ips}:g" \
		-e "s:%%MYBRIDGE%%:${mybridge}:g" \
		${distsharedir}/vpc-rc.tpl > /usr/local/etc/rc.d/cbsd-vpc-${vpc_name}
	${CHMOD_CMD} +x /usr/local/etc/rc.d/cbsd-vpc-${vpc_name}
	/usr/sbin/service cbsd-vpc-${vpc_name} enable
	return 0
}

destroy_bridge()
{
	local _node_list _peer_network _myip= _dbpath _res _ret _bridge_id

	_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	_peer_network=$( cbsdsqlro ${_dbpath} "SELECT dst_node FROM vpc_peers LIMIT 1" )
	[ -z "${_peer_network}" ] && err 1 "${N1_COLOR}peer not initialized, please use first: ${N2_COLOR}cbsd vpc mode=init_peers vpc_name=${vpc_name}${N0_COLOR}"

	. ${subrdir}/vnet.subr

	mybridge=
	if ! mybridge=$( get_my_device vpc vpc-${vpc_name} ); then
		${ECHO} "${N1_COLOR}${CBSD_APP} failed to get bridge: ${mybridge}${N0_COLOR}"
		return 1
	fi

	${IFCONFIG_CMD} ${mybridge} > /dev/null 2>&1
	_ret=$?
	if [ ${_ret} -ne 0 ]; then
		err 1 "${N1_COLOR}no such bridge interface: ${N2_COLOR}${mybridge}${N0_COLOR}"
	fi

	${IFCONFIG_CMD} ${mybridge} destroy
	return 0
}


# ifconfig vxlan23 create vxlanid 23 vxlanlocal 2a05:3580:d800:20f7::1 vxlanremote 2a01:4f8:241:500b::1 inet 10.10.23.3/24 mtu 1450 up
sync_vpc()
{
	local _node_list _peer_network _ret
	_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	_peer_network=$( cbsdsqlro ${_dbpath} "SELECT dst_node FROM vpc_peers LIMIT 1" )
	[ -z "${_peer_network}" ] && err 1 "${N1_COLOR}peer not initialized, please use first: ${N2_COLOR}cbsd vpc mode=init_peers vpc_name=${vpc_name}${N0_COLOR}"

	_node_list=$( cbsdsqlro ${_dbpath} SELECT node_member FROM vpc LIMIT 1 | ${AWK_CMD} '{printf $1}' | ${TR_CMD} ',' ' ' )

	for i in ${_node_list}; do
		nodescp ${_dbpath} ${i}:var/db/vpc/${vpc_name}.sqlite tryoffline=1 verbose=0
		_ret=$?
		if [ ${_ret} -ne 0 ]; then
			${ECHO} "${N1_COLOR}unable to sync vpc to: ${N2_COLOR}${i}${N0_COLOR}"
			err 1 "${N1_COLOR}try: ${N2_COLOR}nodescp ${_dbpath} ${i}:var/db/vpc/${vpc_name}.sqlite tryoffline=1 verbose=1${N0_COLOR}"
		fi
	done

	return 0
}

# check that all nodes see everyone
# this is a very tough test, 
# but we don’t have a common bus yet
recursive_node_check()
{
	local _test_str _nodes _node_list _failed_hosts

	local _dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	_node_list=$( cbsdsqlro ${_dbpath} SELECT node_member FROM vpc LIMIT 1 | ${AWK_CMD} '{printf $1}' | ${TR_CMD} ',' ' ' )
	_node_list="${nodename} ${_node_list}"

	for i in ${_node_list}; do
		if [ -z "${_nodes}" ]; then
			_nodes="${i}"
		else
			_nodes="${_nodes},${i}"
		fi
	done

	_test_str="/usr/local/bin/cbsd rexe tryoffline=1 node=${_nodes} date>/dev/null 2>&1"
	printf "${N1_COLOR}check nodes connection...${N0_COLOR}"

	_failed_hosts=

	for i in ${_node_list}; do
		rexe node=${i} cmd="\"${_test_str}\""
		_ret=$?
		_all_ret=$(( _all_ret + _ret ))
		[ ${_ret} -ne 0 ] && _failed_hosts="${_failed_hosts} ${i}"
	done

	if [ ${_all_ret} -eq 0 ]; then
		${ECHO} "${N2_COLOR}online${N0_COLOR}"
	else
		${ECHO} "${W1_COLOR}failed ${N1_COLOR}(check nodes on following host: ${_failed_hosts})${N0_COLOR}"
	fi
	return ${_all_ret}
}

deploy()
{
	local _test_str _nodes _node_list _failed_hosts _ret

	local _dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	recursive_node_check

	_node_list=$( cbsdsqlro ${_dbpath} SELECT node_member FROM vpc LIMIT 1 | ${AWK_CMD} '{printf $1}' | ${TR_CMD} ',' ' ' )
	_node_list="${nodename} ${_node_list}"

	${ECHO} "${N1_COLOR}  deploy: ${N2_COLOR}init_peers${N0_COLOR}"
	vpc mode=init_peers vpc_name=${vpc_name} 2>/dev/null
	_ret=$?
	[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}deploy failed${N0_COLOR}"
	${ECHO} "${N1_COLOR}  deploy: ${N2_COLOR}sync${N0_COLOR}"
	vpc mode=sync vpc_name=${vpc_name} 2>/dev/null
	_ret=$?
	[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}deploy failed${N0_COLOR}"

	${ECHO} "${N1_COLOR}  deploy: ${N2_COLOR}init_vxlan${N0_COLOR}"
	for i in ${_node_list}; do
		${ECHO} "${N1_COLOR}    node: ${N2_COLOR}${i}${N0_COLOR}"
		rexe node=${i} /usr/local/bin/cbsd vpc mode=init_vxlan vpc_name=${vpc_name} 2>/dev/null
		_ret=$?
		[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}deploy failed${N0_COLOR}"
	done

	${ECHO} "${N1_COLOR}  deploy: ${N2_COLOR}init_bridge${N0_COLOR}"
	for i in ${_node_list}; do
		${ECHO} "${N1_COLOR}    node: ${N2_COLOR}${i}${N0_COLOR}"
		rexe node=${i} /usr/local/bin/cbsd vpc mode=init_bridge vpc_name=${vpc_name} 2>/dev/null
		_ret=$?
		[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}deploy failed${N0_COLOR}"
	done

	return 0
}

destroy()
{
	local _test_str _nodes _node_list _failed_hosts _ret

	local _dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
	[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

	recursive_node_check

	_node_list=$( cbsdsqlro ${_dbpath} SELECT node_member FROM vpc LIMIT 1 | ${AWK_CMD} '{printf $1}' | ${TR_CMD} ',' ' ' )
	_node_list="${nodename} ${_node_list}"

	${ECHO} "${N1_COLOR}  destroy: ${N2_COLOR}destroy_bridge${N0_COLOR}"
	for i in ${_node_list}; do
		${ECHO} "${N1_COLOR}    node: ${N2_COLOR}${i}${N0_COLOR}"
		rexe node=${i} /usr/local/bin/cbsd vpc mode=destroy_bridge vpc_name=${vpc_name} 2>/dev/null
		_ret=$?
		if [ ${force} -ne 1 ]; then
			[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}remote destroy bridge failed. Add ${N2_COLOR}force=1${N1_COLOR} to destroy locally${N0_COLOR}"
		fi
	done

	${ECHO} "${N1_COLOR}  deploy: ${N2_COLOR}destroy_vxlan${N0_COLOR}"
	for i in ${_node_list}; do
		${ECHO} "${N1_COLOR}    node: ${N2_COLOR}${i}${N0_COLOR}"
		rexe node=${i} /usr/local/bin/cbsd vpc mode=destroy_vxlan vpc_name=${vpc_name} 2>/dev/null
		_ret=$?
		if [ ${force} -ne 1 ]; then
			[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}remote destroy vxlan failed. Add ${N2_COLOR}force=1${N1_COLOR} to destroy locally${N0_COLOR}"
		fi
	done

	return 0
}

available_properties="vpc_name node_member peer_network"

case "${mode}" in
	init)
		# determine properties
		for i in ${available_properties}; do
			_val=
			eval _val="\$$i"
			 [ -z "${_val}" ] && err 1 "${N1_COLOR}${CBSD_APP}: ${N2_COLOR}${i}= ${N1_COLOR}is mandatory${N0_COLOR}"
		done
		init_vpc
		;;
	list)
		list_vpc
		;;
	init_peers)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		init_peers
		;;
	init_vxlan)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		init_vxlan
		;;
	destroy_vxlan)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		destroy_vxlan
		;;
	init_bridge)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		init_bridge
		;;
	init_rc)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		init_rc
		;;
	destroy_bridge)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		destroy_bridge
		;;
	remove|clean)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
		[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"
		${RM_CMD} -f ${_dbpath}
		${ECHO} "${N1_COLOR}vpc removed: ${N2_COLOR}${vpc_name}${N0_COLOR}"
		;;
	sync)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		sync_vpc
		;;
	deploy)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		deploy
		;;
	destroy)
		[ -z "${vpc_name}" ] && err 1 "${N1_COLOR}${CBSD_APP} ${N2_COLOR}vpc_name= ${N1_COLOR}is mandatory${N0_COLOR}"
		destroy
		_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
		${RM_CMD} ${_dbpath}
		;;
	*)
		err 1 "${N1_COLOR}unknown mode: ${N2_COLOR}${mode}${N1_COLOR}, available: ${N2_COLOR}init,destroy,list,init_peers,init_vxlan,destroy_vxlan,sync,init_bridge,destroy_bridge${N0_COLOR}"
esac

exit 0
