#!/bin/sh -
env="$HOME/.config/globusctl/.env"; test -r "$env" && . "$env"
export GLOBUS_CLI_CLIENT_ID GLOBUS_CLI_CLIENT_SECRET
# dependency: jq, globus

gcs=globus-connect-server

key=/root/deployment-key.json	# might have to install from CyberArk
ports='50000 51000'		# also the default

urlhead=https://app.globus.org/file-manager/collections/
urltail=/overview/subscription
dochead=https://docs.globus.org/
trblurl=${dochead}globus-connect-server/v5.4/troubleshooting-guide

n1='UMich ARC'; t='	'; n2=' Volume Collection'
coll_umarc="
COLL_XX_T${t}8c185a84-5c61-4bbc-b12b-11430e20010f$t$n1 Non-Sensitive Turbo$n2
COLL_XX_L${t}824d7a3d-c8dc-42eb-aa8f-c9cbf5101669$t$n1 Non-Sensitive Locker$n2
COLL_XX_D${t}ab65757f-00f5-4e5b-aa21-133187732a01$t$n1 Non-Sensitive Data Den$n2
COLL_XP_T${t}aaf9c89f-9534-4080-bfc5-3627c6cb3388$t$n1 Sensitive Turbo$n2
COLL_XP_D${t}74f2f824-eadf-4422-a56b-b7596530267d$t$n1 Sensitive Data Den$n2
COLL_GL$t${t}454f457e-a41b-4807-8775-d132f15a228f${t}umich#greatlakes
COLL_LH$t${t}3242c149-a2b9-4dba-9406-ae3717981621${t}umich#lighthouse
COLL_A2$t${t}77f232f8-2ef4-4f92-a339-a6a74c1cf76f${t}umich#armis2 v2"
ep_umarc="
EP_XX$t${t}3273b6e3-a42b-487a-9a7e-ee76bb8e03ff${t}ARC Non-Sensitive Data
EP_XP$t${t}5bb2c248-d72b-42ce-90ce-f5d95d5a3a39${t}ARC Sensitive Data
EP_GL$t${t}4c984b40-a0b2-4d9e-b132-b32735905e23$t$n1 GreatLakes cluster
EP_LH$t${t}158b6521-e390-46b1-89d9-cb5585f20924$t$n1 Lighthouse storage endpoint
EP_A2$t${t}9b4a1a9d-d406-4eb6-b20f-91e7e72f2c1b$t$n1 Armis2 Cluster Endpoint v2"

# globusctl - convenience front-end for common Globus admin operations
# Steve Kinzler, steve@kinzler.com, Jun 24/Jan 25
# https://kinzler.com/me/home.html#unix

# NOTE: We assume one non-10.* ip# on a Globus server.

db=/var/lib/globus-connect-server/gcs-manager/gcs54.db	# ~gcsweb/gcs54.db

case "$1" in
''|-h|--help)	echo "usage: $0 op [ arg ... ]" 1>&2
		$0 -L | sed 's/^/	/'      1>&2;		    exit 1;;
-L)		sed -n 's/#FUNCXX\ //p' `which $0` |
			sed 's/^#//; s/|/,/g; s/)	/	/'; exit;;
-l)		$0 -L | sed 's/[ 	].*//' | tr , '\012';	    exit;;

###############################################################################

umarc|arc)	#FUNCXX list U-M ARC Globus mapped collections and endpoints
	     echo "$coll_umarc$ep_umarc" | sed '/^$/d';			 exit;;
sh)		#FUNCXX output sh code for collection and endpoint variables
	     "$0" arc | sed 's/		*/=/; s/	.*//; s/^/export /'
									 exit;;
csh)		#FUNCXX output csh code for collection and endpoint variables
	     "$0" arc | sed 's/		*/ /; s/	.*//; s/^/setenv /'
									 exit;;

version)		#FUNCXX output the software version
	exec $gcs --version;;

login)		#FUNCXX login for this homedir
	exec $gcs login localhost;;
reauth)		#FUNCXX reauthorize the login for this homedir
	exec $gcs login --force localhost;;
logout)		#FUNCXX logout for this homedir
	exec $gcs logout;;
whoami)		#FUNCXX output current identities, credentials and sessions
	     $gcs whoami
	     $gcs user-credentials list
	exec $gcs session show;;

data_access)	#FUNCXX UUID - output command to fix access to the collection
	urn='urn:globus:auth:scope:transfer.api.globus.org'
	url='https://auth.globus.org/scopes'
	exec echo "globus session consent '$urn:all[*$url/$2/data_access]'";;

###############################################################################

endpoint)	#FUNCXX output endpoint information
	exec $gcs endpoint show;;
endpoint_uuid)	#FUNCXX [ EP ] - output endpoint UUID
	case "$2" in '') $gcs endpoint show | sed -n 's/^ID: *//p'; exit;; esac
	echo "$ep_umarc" | grep "$2" | awk '{ print $2 }';		 exit;;
roles)		#FUNCXX list endpoint roles
	shift; exec $gcs endpoint role list "$@";;
nodes)		#FUNCXX list nodes
	shift; $gcs node list "$@" | sed '/ '`"$0" ip`' /s/$/ */';	 exit;;
nodes_api)	#FUNCXX [ UUID FMT ] - list nodes on the given endpoint
	ep="${2:-`$0 endpoint_uuid`}"; fmt="${3:-unix}"
	globus api gcs -F "$fmt" "$ep" GET /nodes;			 exit;;
	# https://docs.globus.org/globus-connect-server/v5/api/openapi_Nodes
config)		#FUNCXX output nodes configuration
	for uuid in `"$0" nodes | sed '1,2d; s/ .*//'`
	do	$gcs node show -F json "$uuid" | jq
	done; exit;;
gateways)	#FUNCXX list and output storage gateways information
	$gcs storage-gateway list
	$gcs storage-gateway list | sed '1,2d; s/^[^|]*| *//; s/ .*//' |
	while read uuid
	do	echo -------
		$gcs storage-gateway show --include-private-policies "$uuid"
	done;								 exit;;
client_id)	#FUNCXX output endpoint client_id from the database
	exec sudo sqlite3 "$db" "SELECT value FROM configuration
		WHERE name='client_id';";;
secret)		#FUNCXX output endpoint secret from the database
	exec sudo sqlite3 "$db" "SELECT value FROM configuration
		WHERE name='client_secret';";;
	# Note: client secrets can be (re)generated as needed at
	#	https://auth.globus.org/v2/web/developers
checkkey)	#FUNCXX verify the client_id and secret from key vs database
	dbcid=`"$0" client_id`;	kycid=`sudo jq -r .client_id "$key"`
	dbsec=`"$0" secret`;	kysec=`sudo jq -r .secret    "$key"`
	ret=0
	test "$dbcid" != "$kycid" && { ret=1$ret
		echo "client_id mismatch ($dbcid in $db, $kycid in $key)"; }
	test "$dbsec" != "$kysec" && { ret=2$ret
		echo "secret mismatch ($dbsec in $db, $kysec in $key)"; }
								    exit $ret;;

ip)		#FUNCXX output the ip# of this node
	ip addr | grep global | perl -pe 's/^\s*(inet\s*)?//i' |
		grep -v '^10\.' | sed 's/\/.*//';			 exit;;
ip-other)	#FUNCXX output the ip# of the other node
	"$0" nodes | sed 1,2d | grep -v " `$0 ip` " | sed 's/  */ /g' |
		cut -d ' ' -f 3;					 exit;;
node)		#FUNCXX output the uuid of this node
	"$0" nodes |		grep    " `$0 ip` " | sed 's/ .*//';	 exit;;
node-other)	#FUNCXX output the uuid of the other node
	"$0" nodes | sed 1,2d | grep -v " `$0 ip` " | sed 's/ .*//; 1q'; exit;;

#enable)		#FUNCXX enable this node
#enable-other)	#FUNCXX enable the other node
#disable)		#FUNCXX disable this node
#disable-other)	#FUNCXX disable the other node
enable|enable-other|disable|disable-other)
	set x `echo $1 | sed 's/-/ -/'`; shift
	"$0" nodes
	$gcs node update --$1 `"$0" node$2`
	exec "$0" nodes;;
#reenable)	#FUNCXX disable then enable this node
#reenable-other)	#FUNCXX disable then enable the other node
reenable|reenable-other)
	set x `echo $1 | sed 's/-/ -/'`; shift
	"$0" nodes; node=`"$0" node$2`
	$gcs node update --disable "$node"
	$gcs node update --enable  "$node"
	exec "$0" nodes;;

###############################################################################

mapped)		#FUNCXX list mapped collections
	exec $gcs collection list --filter mapped-collections;;

# Note: `globus gcs collection list --limit ...` can also be used,
#       but lacks "Created" and "Last Access" columns

list)		#FUNCXX [ PATT ] - list collections, limited to any pattern
	case "$#" in
	1)	shift; $gcs collection list "$@" | sed 's/  *$//';;
	*)	"$0" list | perl -ne 'print if $.==1 || $.==2 || m<'"$2"'>';;
	esac; exit;;

json)		#FUNCXX PATT - dump json for collections matching pattern
	"$0" list "${2:-XNoSuuchX}" | sed 1,2d |
		perl -pe "s| .*||; s|^|'$0' |" | sh; exit;;
????????-????-????-????-????????????)
	$gcs collection show -F json --include-private-policies "$1" |jq;exit;;

url)		#FUNCXX UUID - output a subscription url for given coll uuid
	exec echo "$urlhead$2$urltail";;

delete)		#FUNCXX UUID - delete the given collection
	exec $gcs collection delete "${2:-ProvideAnUUID}";;

perms)		#FUNCXX UUID - output permissions json for the given collection
	exec globus api transfer \
		    GET /endpoint/"${2:-ProvideAnUUID}"/access_list;;
	# https://docs.globus.org/api/transfer/permissions

###############################################################################

# NOTE: After initial installation, this may be needed again after a server
#	reload or all nodes in an endpoint being disabled.
register)	#FUNCXX register, setup and activate this node
	sudo $gcs node setup --ip-address `"$0" ip` \
		--deployment-key "$key" --incoming-port-range $ports;	 exit;;
# NOTE: Later adjustments can be made, eg:
#	sudo $gcs node update --incoming-port-range $ports <UUID>
# See also https://github.com/umich-arc/ARC-Wiki/wiki\
#		/Re%E2%80%90registering-a-Globus-node-server

cxion)		#FUNCXX test the network connection to Globus services
	exec nc -z -v -w 5 relay.globusonline.org 2223;;

audit)		#FUNCXX output reference for HA audits
	exec echo See $dochead$gcs/v5/reference/audit/;;

check)		#FUNCXX check all collections
	$gcs collection check | grep -v 'No errors detected';		 exit;;
check_api)	#FUNCXX [ UUID ] - check all collections on the given endpoint
	globus api gcs -F unix "${2:-`$0 endpoint_uuid`}" \
	       GET /collections/check | perl -F'\t' -ane \
		   'print "$F[1]\n    $F[2]\n" if $F[0] =~ /^check_re/'; exit;;
	# https://docs.globus.org/globus-connect-server/v5/api\
	# /openapi_Collections/#checkCollections
checkpaths)	# [ FILE ] - check partial validity of given collection paths
	IFS='	'; cat "$2" |	# input in globus/guests file format
	while read uuid name path user creat used; do
		path=`echo "$path" | sed 's/^[A-Z][A-Z0-9]://'`
		vol=` echo "$path" | perl -pe '
s:^(/nfs/(dataden|locker|mm-isilon|turbo)/[^/]+).*$:$1: && next;
s:^(/(dept|gpfs|home|nfs|scratch)/[^/]+).*$:$1:'`
		test -e "$vol" && continue
		(cat; echo) <<EOF
$uuid
    Collection $uuid is not currently
    valid. The volume "$vol" for the path
    "$path" does not exist.
    [$name	$user	$creat	$used]
EOF
	done;								 exit;;
checkall)	# run all collection checks
	"$0" check; shift; exec "$0" checkpaths "$@";;

# Note: can add like "--collection COLLECTIONUUID"
diag)		#FUNCXX run self-diagnostic
	sudo $gcs self-diagnostic ${1+"$@"} |
		sed 's/"secret":.*/"secret": REDACTED/';		 exit;;
debug)		#FUNCXX how to set debug logging levels
	exec cat <<EOF
See $trblurl#obtaining-debug-log-events
GridFTP: place only "log_level ALL" in /etc/gridftp.d/z_logging
GCS Mgr: place only "GCS_MANAGER_LOG_LEVEL=DEBUG"
	 in any /etc/sysconfig/gcs_manager and run:
		systemctl restart gcs_manager.service
Use as approriate:	$0 marklogs [ begin | end ]
EOF
	;;

###############################################################################

active)		#FUNCXX output active status of globus services
	for service in httpd gcs_manager globus-gridftp-server \
		       gridftp-server-restarter globus-oidc
	do
		echo "$service is `systemctl is-active $service`"
	done; exit;;
kick)		#FUNCXX restart the gridftp service
	exec sudo systemctl restart globus-gridftp-server;;

logs)		#FUNCXX [ LS_ARGS ] - ls most useful current globus log files
	shift; exec sudo ls "$@" /var/log/gridftp.log \
			/var/log/gridftp-audit.log \
			/var/log/globus-connect-server/gcs-manager/gcs.log \
			/var/log/httpd/globus_access_log \
			/var/log/httpd/error_log /var/log/messages;;

marklogs)	#FUNCXX [ TEXT ] - append a marker to globus cxion log files
	shift; args=`echo "$@" | sed "s/'//g"`
	mark="#-------- MARK `date` $args --------"
	exec sudo sh -c "
	  echo '$mark' >> /var/log/gridftp.log
	  echo '$mark' >> /var/log/globus-connect-server/gcs-manager/gcs.log";;

speeds)		#FUNCXX output xfer speeds from gridftp.log input
	exec perl -ne 'BEGIN { use Time::Local }
	s/^.+(\[\d+\])/$1/;	# ignore overwritten partial log line
	# warning: wont get full filename if it contains whitespace
	($e, $b, $f, $n) = / ^\[ .* \s::\sTransfer\sstats: .*
		\sDATE=(\d{14}(?:\.\d*)?)\s .* \sSTART=(\d{14}(?:\.\d*)?)\s .*
		\sFILE=(\S*) .* \sNBYTES=(\d+)\s .*$ /xi	or next;
	$p = q(^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})(.*)$);
	eval {	$e =~ /$p/; $e = timelocal($6, $5, $4, $3, $2-1, $1) + $7;
		$b =~ /$p/; $b = timelocal($6, $5, $4, $3, $2-1, $1) + $7; };
	chomp($@), print STDERR "$@: with line: $_" if $@;
	print int($b),"\t$n\t", $n / ($e - $b) / 1024 / 1024, "\t$f\n"
		if $n && $e - $b > 0';;
speedsbin)	#FUNCXX output speeds input binned by hour and /nfs environment
	min="${2:-1048576}"
	exec awk -F'\t' '$2 >= '"$min"' && match($4, /^\/nfs\/([^\/]+)/, a) {
	$4=a[1]; cmd="date -d @"$1" +%Y-%m-%d_%H"; cmd|getline dt; close(cmd);
	print dt "\t" $3 "\t" $4 }';;
speedsavg)	#FUNCXX output binned speeds input averaged per bin
	exec perl -F'\t' -lane '$k = "$F[0]\t$F[2]"; $s{$k} += $F[1]; $n{$k}++;
	END { for (keys %s) { print "$_\t", $s{$_} / $n{$_} } }';;
speedsplot)	#FUNCXX plot averaged binned speeds input for turbo and dataden
	tmp=/tmp/gp$$; tmpt=/tmp/gpt$$; tmpd=/tmp/gpd$$
	trap "rm -f $tmp $tmpt $tmpd; exit" 0 1 2 13 15; cat > $tmp
	grep turbo < $tmp | sort > $tmpt; grep dataden < $tmp | sort > $tmpd
	exec gnuplot -p -e 'set datafile separator "\t"; set xdata time;
	   set timefmt "%Y-%m-%d_%H"; set format x "%m-%d\n%H:00";
	   plot "'"$tmpt"'" using 1:3 with linespoints lw 3 title "turbo",
		"'"$tmpd"'" using 1:3 with linespoints lw 3 title "dataden"';;
esac

###############################################################################

export PATH="$PATH:`dirname $0`"
export USE_STOR_DB=true
guests=`stordb 2> /dev/null`/globus/guests

case "$1" in
coll2vol)	#FUNCXX [ PATT ] - output volumes hosting matching collections
	exec perl -F'\t' -ane 'print $F[2]
				 if "@F[0,1]" =~ m~'"$2"'~' "$guests";;
vol2coll)	#FUNCXX [ PATT ] - output collections hosted on matching vols
	exec perl -F'\t' -ane 'print if $F[2] =~ m~'"$2"'~' "$guests";;

*)	echo "$0: invalid op ($1)" 1>&2; exec "$0" -h;;
esac