#!/usr/local/bin/perl
#
#Script to send emails to PoCs re archived data/home directories
#which are to be deleted soon.
#Expected to be run from cron job
use strict;
use warnings;

use lib "/cell_root/software/HPCC/lib/site_perl";

use Glue::HPCC::Cluster;

use Glue::HPCC::Project::Project;
#use Glue::HPCC::Project::ProjectType;
use Glue::HPCC::Project::Person;
use Glue::HPCC::Project::PersonProject;
#use Glue::HPCC::Project::LastEmail;

use Glue::HPCC::Utils::FileUtils qw(
	parse_archive_metainfo_file
	delete_archived_data_dir
	delete_archived_home_dir
);
use Glue::HPCC::Utils::MailUtils qw(send_template_email);

use Getopt::Long;
use Time::ParseDate;
use POSIX qw(strftime);
use Carp qw(carp croak);
use Cwd;


my $DEBUG;
my $DEFAULT_GRACEDAYS = 35;

sub usage()
{	
	print STDERR <<EOF;
$0: Cron job to alert PoCs to pending data/home dir deletions

Usage:
$0 [ --debug ] [ --dryrun ] [ -[no]mail ] \
	[ --maxdays MAXDAYS ] [ --grace GRACEDAYS ] \
	[ --expired-mode EXPMODE ] \
	[ --noprojs-mode NOPROJMODE ] \
	[ --nopocs-mode NOPOCMODE ] \
	[ --bad-mode BADMODE ]

This scans all of the 'pending-deletion' directories on the cluster
for metainfo ".readme" files, and will generate email to the 
appropriate PoCs regarding any directories that are slated to be
deleted.  It also will optionally email system admins and/or delete
archives eligible for deletion.

GRACEDAYS specifies (in days) a grace period between expiration of
archive and earliest date can be deleted due to expiration.  Default
for GRACEDAYS is $DEFAULT_GRACEDAYS.  MAXDAYS
specifies (in days) a limit on warning emails; if MAXDAYS is 90,
warnings will only be sent regarding archives expiring in next 90 days.
By default, warnings will not be limited in that way (effectively infinite
MAXDAYS).

Archives can fall into one of 4 categories:
1) expired.  Archive expiration (from metainfo) is at least GRACEDAYS days ago.
2) project-less. No projects are listed for this archive (in metainfo)
3) no objecting PoCs.  All PoCs for all projects listed for this archive
have OKed the deletion of archive (indicated by presence on ExcludePoCs list
in metainfo)
4) bad archives.  Something invalid in the metadata.
5) standard archived directories
Behavior of script depends on type of archive.  The first three cases are
normally eligible for deletion.  Behavior of the script on archives of the 
first three types is controlled by EXPMODE, NOPROJMODE, NOPOCMODE, and BADMODE resp.
These can be set to one of:
	EMAIL: emails system staff to inform of status.  No other action taken.
	DELETE: deletes the archive.  Not valid for BADMODE
	IGNORE: does nothing (not deleted, no email)
	POC: email POCs (just as "standard" case).  No other action taken.
		Only valid for EXPMODE.
The default for all four is 'EMAIL'.

For the standard case, all PoCs associated with projects the user belonged
to (i.e. projects in metainfo for archive) will be sent an email warning them
of the pending deletion of the archive.

Emails are collected, so each affected PoC will receive one email with info
on all archives pending deletion he is associated with.

In --dryrun mode, no emails or deletions are actually done, but
the commands/emails that would have been sent are sent to STDERR.
If --nomail is given, no emails will be actually sent, but deletions
may occur.

EOF
}

sub default_metainfo_dirs_from_hpc($)
#Default metadirs from the cluster 
{	my $hpc = shift;
	my $me = 'default_metainfo_dirs_from_hpc';

	croak "$me: Missing/invalid required parm hpc at "
		unless $hpc && ref($hpc);

	my $midirs = [];
	my $pdsuff = 'pending_deletion';

	my $rootdirs = $hpc->lustre_filesystems || [];
	my $tmp = $hpc->nfs_data_filesystems || [];
	push @$rootdirs, @$tmp if $tmp && ref($tmp) eq 'ARRAY';
	$tmp = $hpc->homeroot_user;
	push @$rootdirs, $tmp if $tmp;
	$tmp = $hpc->homeroot_admin;
	push @$rootdirs, $tmp if $tmp;

	foreach my $tmpdir (@$rootdirs)
	{	$tmp = "${tmpdir}/${pdsuff}";
		push @$midirs, $tmp
			if -d $tmpdir && -d $tmp;
	}

	#Remove any duplicates
	my %temp = map { $_ => $_ } @$midirs;
	$midirs = [ values %temp ];

	carp "$me: Unable to find any $pdsuff dirs from HPC cluster at "
		unless scalar(@$midirs);

	if ( $DEBUG )
	{	print STDERR "[DEBUG] Using metainfo directories:\n";
		foreach my $tmpdir (@$midirs)
		{	print STDERR "[DEBUG]	$tmpdir\n";
		}
	}
	return $midirs;
}

sub get_all_metainfo_files($)
#Takes a list ref of directories to look in, returns a list ref of
#paths to all *.readme files in all given directories.
{	my $metadirs = shift || [];
	my $me = 'get_all_metainfo_files';

	my $metafiles = [];
	MDIR: foreach my $mdir (@$metadirs)
	{	my $MDIR;
		next MDIR unless -d $mdir;
		
		opendir($MDIR,$mdir) or die "Error opening metadir $mdir: $! at";
		my @files = readdir($MDIR);
		closedir($MDIR);

		@files = grep /\.readme$/, @files;
		@files = grep { -f "$mdir/$_" } @files;
		@files = map "${mdir}/$_", @files;
		push @$metafiles, @files;
	}

	#Remove any duplicates
	my %temp = map { $_ => $_ } @$metafiles;
	$metafiles = [ values %temp ];

	return $metafiles;
}

sub prune_missing_metainfo_files($;$)
#Takes list refs of metainfo files and metainfo directories, and
#returns a list ref of full paths to metainfo files.
#For each metainfo file in first list ref:
#	if it is NOT an absolute path (i.e. does not start with /)
#		add fullpath relative to . if exists
#		loop through each metadir and add fullpath relative
#			to mdir if it exists
#		warn if nothing matched for that entry
#	otherwise add the absolute path if it exists (and warn otherwise)
{	my $metafiles = shift;
	my $metadirs = shift || [];
	my $me = 'prune_missing_metainfo_files';

	die "$me: Missing required parm 'metafiles' at " unless $metafiles;
	die "$me: Invalid val '$metafiles' for 'metafiles', expecting list ref, at "
		unless ref($metafiles) eq 'ARRAY';
	die "$me: Empty list given for 'metafiles' at " unless scalar(@$metafiles);

	my @metafiles = ();
	my $cwd = cwd;

	foreach my $mfile (@$metafiles)
	{   if ( $mfile =~ '^/' )
	    {	#Got an absolute path
		if ( -f $mfile )
		{	push @metafiles, $mfile;
		} else
		{	warn "$me: File $mfile not found, skipping.\n";
		}
	    } else
	    {	my $tmp = "${cwd}/${mfile}";
		my $found = 0;
		if ( -f $tmp )
		{	push @metafiles, $tmp;
			$found++;
		}
		foreach my $mdir (@$metadirs)
		{	$tmp = "${mdir}/${mfile}";
			if ( -f $tmp )
			{	push @metafiles, $tmp;
				$found++;
			}
		}
		unless ( $found )
		{    warn "$me: File $mfile not found anywhere, skipping.\n";
		}
	    }
	}

	return \@metafiles;
}

sub read_metainfo_files($)
#Takes a list ref of metainfo files (full paths to file) and reads
#then, returning as metadata hash refs.
#
#On success, returns a hash ref with 2 keys:
#	good: value is a list ref consisting of metadata
#		hash refs of metadata files successfully read.
#	bad: value is a hash ref.  Keys are the names
#		of metadata files which were not successfully
#		read, and value is the error from attempting
#		to read them.
#
#On non-fatal errors, returns a non-ref error string.
#
{	my $metafiles = shift || [];
	my $me = 'read_metainfo_files';

	my $metainfo = [];
	my $badfiles = {};

	MFILE: foreach my $mfile (@$metafiles)
	{	my $minfo = parse_archive_metainfo_file($mfile);
		unless ( $minfo && ref($minfo) eq 'HASH' )
		{   #warn "$me: Skipping invalid metainfo file $mfile: $minfo\n";
		    $badfiles->{$mfile} = $minfo;
		    next MFILE;
		}

		#Make sure record has name of directory archived,
		#the owner, and an expiration tstamp
		#If any missing, add to badfiles
		unless ( $minfo->{DirectoryArchived} )
		{   #warn "$me: Skipping invalid metainfo file $mfile: " .
		    #	"no DirectoryArchived field\n";
		    $badfiles->{$mfile} = "no DirectoryArchived field";
		    next MFILE;
		}
		unless ( $minfo->{DirectoryOwnerUsername} )
		{   #warn "$me: Skipping invalid metainfo file $mfile: " .
		    $badfiles->{$mfile} = "no DirectoryOwnerUsername field";
		    next MFILE;
		}
		unless ( defined $minfo->{ExpirationTstamp} )
		{   #warn "$me: Skipping invalid metainfo file $mfile: " .
		    $badfiles->{$mfile} = "no ExpirationTstamp field";
		    next MFILE;
		}
		
		#Looks like we got good metainfo, add it to the list
		push @$metainfo, $minfo;
	}

	return { good => $metainfo, bad => $badfiles};
}

sub categorize_metainfo_files($$$)
#Takes a Glue::HPCC::Cluster instance, a list ref of metadata
#hashes, and an optional options hash, and sorts the metadata
#by "category" and/or poc.
#
#Options is a hash ref of options controlling behavior.  Defaults
#to empty hash.  Recognized keys are:
#	now => Time (in seconds from 1970) to use as "now"
#	gracedays => grace period after expiration, in days.
#		Default is $DEFAULT_GRACEDAYS
#	maxdays => ignore metadata with expiration dates more than 
#		maxdays into the future.  Default is undef, which
#		means do ignore any metadata.
#
#Metadata can fall into 5 categories:
#1) bad/incomplete. Missing required fields (ExpirationTstamp, etc)
#2) noprojects: There are no projects listed for DirectoryOwnerProjects
#3) expired: The expiration date is more than $gracedays before $now
#4) nopocs: All the PoCs for all projects in DirectoryOwnerProjects
#	are in the ExcludePoCs list.  (To maintain disjointness of
#	categories, we require DirectoryOwnerProjects to be non-empty
#	for this category).
#5) standard archives
#
#This script will classify each metadata info hash for each archive
#into one of the above 5 categories, which are collected as follows.
#Bad, noprojects, and nopocs metadata are each put into their own
#respective list ref.
#
#For expired and standard metadata, we get usernames of all PoCs for
#all projects in DirectoryOwnerProjects, excluding those in ExcludePoCs.
#We then create hashes (one for expired, one for standard), keyed on
#PoC name, with values being list ref of corresponding metadata hashes.
#
#We return a hash ref with keys:
#	bad => a hash ref, keyed on file name of bad metadata info files,
#		value is the reason rejected.
#	noprojects => the list ref of metadata hash refs w/out projects
#	nopocs => the list ref of metadata hash refs w/out nonexcl PoCs
#	expired => the list ref of metadata hash refs that expired (plus grace)
#	expbypoc => the hash ref (keyed on PoC username) of expired
#		metadata.  Note that "expired" means $now is more than
#		$gracedays days past the expiration date
#	bypoc => the hash ref (keyed on PoC username) of standard
#		metadata hashes. 
#
#NOTE: If MAXDAYS is defined and non-zero, bypoc will OMIT any metadata 
#whose expiration is more than MAXDAYS in the future.
#
#NOTE: In all of the above, "expired" means both the expiration date
#and the grace period have passed (relative to $now).
{	my $hpc = shift;
	my $metadata = shift || [];
	my $options = shift || {};
	my $me = 'categorize_metainfo_files';

	die "$me: Missing required parameter 'hpc' at " unless $hpc;
	die "$me: Invalid value '$hpc' for hpc, expect Glue::HPCC::Cluster"
		. " instance at " unless ref($hpc);
	die "$me: Invalid metadata list ref '$metadata' at" 
		unless $metadata && ref($metadata) eq 'ARRAY';

	my $now = delete $options->{now} || time;
	my $maxdays = delete $options->{maxdays};
	my $gracedays = delete $options->{gracedays};
	$gracedays = $DEFAULT_GRACEDAYS unless defined $gracedays;
	die "$me: Invalid negative gracedays '$gracedays' at "
		if $gracedays < 0;
	if ( %$options)
	{	my $tmp = join ", ", (keys %$options);
		die "$me: Unrecognized options [ $tmp ], aborting at ";
	}

	my $bad = {};
	my $noprojs = [];
	my $nopocs = [];
	my $expired = [];
	my $expbypoc = {};
	my $bypoc = {};

	my $gracesecs = $gracedays * 24 * 60 * 60;
	my $maxsecs = 0;
	$maxsecs = $maxdays * 24 * 60 * 60 if defined $maxdays;

	#read_metainfo_files ensures has DirectoryArchived, DirectoryOwner,
	#and ExpirationTstamp
	METADATA: foreach my $mdata (@$metadata)
	{	#Process record
		my $exptstamp = $mdata->{ExpirationTstamp};
		#Check anyway
		unless ( defined $exptstamp)
		{	my $mfile = $mdata->{MetadataSource};
			$bad->{$mfile} = 'No expiration tstamp';
			next METADATA;
		}
		my $effexp = $exptstamp + $gracesecs;
		
		#Handle noprojs
		my $projs = $mdata->{DirectoryOwnerProjects};
		unless ($projs && ref($projs) eq 'ARRAY' && scalar(@$projs) )
		{	push @$noprojs, $mdata;
			next METADATA;
		}

		#Collect all PoCs for archive
	 	my $exclude = $mdata->{ExcludePoCs} || [];
		my %excluded = map { $_ => undef } @$exclude;
		my @pocs;
	    	foreach my $proj (@$projs)
	    	{	my $tmppocs = Glue::HPCC::Project::PersonProject->get_pocs_for_project(
				$hpc, $proj);
			die "$me: Error getting pocs for project $proj: $tmppocs, at "
				unless $tmppocs && ref($tmppocs) eq 'ARRAY';
			push @pocs, @$tmppocs;
		}
		#Remove dups
		my %temp = map { $_ => undef } @pocs;
		@pocs = keys %temp;
		#Remove excluded
		@pocs = grep { ! exists $excluded{$_} } @pocs;

		unless ( @pocs )
		{	push @$nopocs, $mdata;
			next METADATA;
		}

		#Are we expired?
		if ( $now > $effexp )
		{	#Expired
			#Add this metadata record to expired list
			push @$expired, $mdata;
			#and Add this metadata record to list for each poc
			foreach my $poc (@pocs)
			{	my $tmplist = $expbypoc->{$poc} || [];
				push @$tmplist, $mdata;
				$expbypoc->{$poc} = $tmplist;
			}
			next METADATA;
		} else
		{	#Not expired
			#Filter out based on MAXDAYS
			if ( defined $maxdays )
			{	#Apply MAXDAYS filter
				if ( $now + $maxsecs < $exptstamp )
				{	#Records expires more than maxdays from now, skip it
					next METADATA;
				}
			}
			#Add to list for poc
			foreach my $poc (@pocs)
			{	my $tmplist = $bypoc->{$poc} || [];
				push @$tmplist, $mdata;
				$bypoc->{$poc} = $tmplist;
			}
			next METADATA;
		}
				
		die "Should not reach here at ";
	}

	my $results = 
	{	bad => $bad,
		noprojects => $noprojs,
		nopocs => $nopocs,
		expired => $expired,
		expbypoc => $expbypoc,
		bypoc => $bypoc,
	};

	return $results;
}

sub sort_metadata_for_poc($$$)
#Takes $hpc instance, username of PoC, and a list ref of 
#metadata hash refs (for given #PoC), 
#and sorts into format desired for email to PoCs.
#
#On success, returns a list ref.  On non-fatal error, a non-ref
#error string.  List ref returned is a list of hash refs, in
#order of reverse expiration date (i.e. records expiring soonest
#are first), with the hash ref having keys:
#	Expiration: date at which everything in hash expires
#		(we purposefully strip off time of day)
#	ExpTstamp. timestamp of above.  Just use any tstamp for
#		any record under this hash, just for sorting purposes
#	Users: list ref (sorted by username) of hash refs for
#		data pending deletion.
#
#Each element of the Users list should again be hash refs, with
#keys:
#	Username: username of user (owner of files to be deleted)
#	UserFullname: fullname of user
#	Projects: list ref containing names of Projects the user
#		was member of and the PoC is a Poc for.
#	Directories: list ref of hash ref of directories owned
#		by user pending deletion
#
#The Directories list consists of hash refs with keys
#	Directory: original path of directory
#	DataType: 'nfs-data', 'lustre-data', or 'home'
{	my $hpc = shift;
	my $poc = shift;
	my $metadata = shift || [];
	my $me = 'sort_metadata_for_poc';

	die "$me: Missing/invalid value for 'hpc' at "
		unless ( $hpc && ref($hpc) );
	die "$me: Missing/invalid value for 'poc' at "
		unless $poc;
	die "$me: Missing value for metadata array at "
		unless ( $metadata);
	die "$me: Invalid value '$metadata' for 'metadata', expecting " .
		"list ref, at " unless ref($metadata) eq 'ARRAY';

	#Temporary storage of results
	my $reshash = {};

	#Get projects poc is PoC of
	my $projs_by_poc = Glue::HPCC::Project::PersonProject->get_projects_user_is_poc_of(
		$hpc, $poc);
	return "$me: Unable to get projects $poc is PoC of: $projs_by_poc"
		unless $projs_by_poc || ref($projs_by_poc) eq 'ARRAY';
	my %projs_by_poc =  map { $_ => undef } @$projs_by_poc;


	MIREC: foreach my $mdata (@$metadata)
	{   my $exptstamp = $mdata->{ExpirationTstamp};
	    unless ( $exptstamp )
	    {	warn "$me: Skipping record missing ExpTstamp at ";
		next MIREC;
	    }
	    my $expdate = strftime("%e %b %Y", localtime($exptstamp) );

	    my $projs = $mdata->{DirectoryOwnerProjects};
	    unless ( $projs && ref($projs) eq 'ARRAY' && scalar(@$projs) )
	    {	warn "$me: Skipping record w/out any projects at ";
		next MIREC;
	    }
	    #Now get intersection between user's and PoC's projects
	    my $commprojs = [ grep { exists $projs_by_poc{$_} } @$projs ];

	    my $owner = $mdata->{DirectoryOwnerUsername}
		|| "UID:" . $mdata->{DirectoryOwnerUID};
	    my $fowner = $mdata->{DirectoryOwnerFullname} || $owner;

	    my $dname = $mdata->{DirectoryArchived};
	    unless ( $dname )
	    {	warn "$me: Skipping record w/out dir name at ";
		next MIREC;
	    }
	    my $dtype = $mdata->{DirectoryType};

	    my $exphash = $reshash->{$expdate};
	    unless ( $exphash && ref($exphash) eq 'HASH' )
	    {	$exphash = {};
		$reshash->{$expdate} = $exphash;
	    }

	    $exphash->{Expiration} = $expdate;
	    $exphash->{ExpTstamp} = $exptstamp;
	    my $byuser = $exphash->{UsersHash};
	    unless ( $byuser && ref($byuser) eq 'HASH' )
	    {	$byuser = {};
		$exphash->{UsersHash} = $byuser;
	    }

	    my $userhash = $byuser->{$owner};
	    unless ( $userhash && ref($userhash) eq 'HASH' )
	    {	$userhash = {};
		$byuser->{$owner} = $userhash;
	    }

	    $userhash->{Username} = $owner;
	    $userhash->{UserFullname} = $fowner;
	    $userhash->{Projects} =  [ @$commprojs ];

	    my $dlist = $userhash->{Directories};
	    unless ( $dlist && ref($dlist) eq 'ARRAY' )
	    {	$dlist = [];
		$userhash->{Directories} = $dlist;
	    }

	    push @$dlist, 
	    {	Directory => $dname,
		DataType => $dtype,
	    };
	}

	#Fix some stuff (convert hashes to lists, etc)
	#Convert UserHash into Users list sorted by username
	foreach my $expdate (keys %$reshash)
	{	my $ehash = $reshash->{$expdate};
		my $uhash = delete $ehash->{UsersHash};
		my @users = values %$uhash;
		@users = sort { $a->{Username} cmp $b->{Username} } @users;
		$ehash->{Users} = [ @users ];
	}

	#Convert reshash hash into list
	my @records = values %$reshash;
	@records = sort { $a->{ExpTstamp} <=> $b->{ExpTstamp} } 
		@records;
	my $results = [ @records ];

	return $results;
}


#-------------------------------------------------------------------
#		Main
#-------------------------------------------------------------------

my $help;
my $maxdays;
my $gracedays = $DEFAULT_GRACEDAYS;
my $cluster;
my $metadirs = [];
my $metafiles = [];
my $dryrun;
my $verbose;
my $now;
#Defaults
my $expmode = 'EMAIL';
my $noprojmode = 'EMAIL';
my $nopocmode = 'EMAIL';
my $badmode = 'EMAIL';

my $res = GetOptions(
	'help!' => \$help,
	'debug!' => \$DEBUG,
	'mail|email!' => sub { my $n = shift; my $v = shift;
		#--mail is basically inverse of --dryrun
		$dryrun = ($v)?0:1;
		},
	'dryrun!' => \$dryrun,
	'verbose!' => \$verbose,
	'maxdays|max-days|max_days=f' => \$maxdays,
	'gracedays|grace|grace-days|grace_days=f' => \$gracedays,
	'cluster=s' => \$cluster,
	'now=s' => \$now,
	'expired-mode|exp-mode|expired_mode|exp_mode|expiredmode|expmode|exp=s' => \$expmode,
	'noprojs-mode|noprojects-mode|noprojs_mode|noprojects_mode|noprojsmode|noprojectsmode|noprojects=s' => \$noprojmode,
	'nopocs-mode|nopocs_mode|nopocsmode|nopocs=s' => \$nopocmode,
	'bad-mode|bad_mode|badmode|bad=s' => \$badmode,
);

unless ( $res ) 
{	usage;
	die "\nError processing options.\n";
}

if ( $help )
{	usage;
	exit 0;
}

if ( scalar(@ARGV) )
{       my $tmp = join ", ", @ARGV;
        usage;
        die "\nError: extraneous arguments [ $tmp ], aborting\n";
}

#Validate mode values

#Expired mode
$expmode = uc $expmode if $expmode;
unless ( $expmode eq 'EMAIL' || $expmode eq 'DELETE' ||
	$expmode eq 'IGNORE' || $expmode eq 'POC' )
{	usage;
	die "\nInvalid noprojects-mode '$expmode', expecting one of "
		. "EMAIL, DELETE, IGNORE, or POC.\n";
}

#Nopocs mode
$nopocmode = uc $nopocmode if $nopocmode;
unless ( $nopocmode eq 'EMAIL' || $nopocmode eq 'DELETE' ||
	$nopocmode eq 'IGNORE' )
{	usage;
	die "\nInvalid nopocs-mode '$nopocmode', expecting one of "
		. "EMAIL, DELETE, or IGNORE.\n";
}

#Noprojs mode
$noprojmode = uc $noprojmode if $noprojmode;
unless ( $noprojmode eq 'EMAIL' || $noprojmode eq 'DELETE' ||
	$noprojmode eq 'IGNORE' )
{	usage;
	die "\nInvalid noprojects-mode '$noprojmode', expecting one of "
		. "EMAIL, DELETE, or IGNORE.\n";
}

#bad mode
$badmode = uc $badmode if $badmode;
unless ( $badmode eq 'EMAIL' || 
	$badmode eq 'IGNORE' )
{	usage;
	die "\nInvalid bad-mode '$badmode', expecting one of "
		. "EMAIL or IGNORE.\n";
}


Glue::HPCC::Utils::MailUtils::verbose($verbose);
Glue::HPCC::Utils::MailUtils::dryrun($dryrun);

#Default cluster if needed
my $hpc = Glue::HPCC::Cluster->new_from_clustername($cluster);
die "\nInvalid cluster $cluster at " if $cluster && ! $hpc && ! ref($hpc);
die "\nError instantiating cluster $hpc " unless $hpc && ref($hpc);
$cluster = $hpc->db_clustername;

if ( defined $now )
{	#Convert to tstamp
	my $new = parsedate($now,
		WHOLE => 1, VALIDATE => 1 );
	unless ( defined $new )
	{	die "Invalid value '$new' for --now; need something " .
			"Time::ParseDate can parse\n";
	}
	$now = $new;
} else
{	$now = time;
}

unless ( $metadirs && ref($metadirs) eq 'ARRAY' && scalar(@$metadirs) )
{	$metadirs = default_metainfo_dirs_from_hpc($hpc);
}

unless ( $metafiles && ref($metafiles) eq 'ARRAY' && scalar(@$metafiles) )
{	unless ( scalar(@$metadirs) )
	{	die "No metainfo files or directories given and unable to default.\n";
	}
	$metafiles = get_all_metainfo_files($metadirs);
}
$metafiles = prune_missing_metainfo_files($metafiles, $metadirs);
unless ( scalar(@$metafiles) )
{	die "No metafiles found.\n";
}

if ( $verbose )
{	my $mdirs = join ", ", @$metadirs;
	my $tmpmaxdays = $maxdays || '<undef>';
	print STDERR <<EOF;
[VERBOSE] In verbose mode
[VERBOSE] metadirs=$mdirs
[VERBOSE] now=$now
[VERBOSE] gracedays=$gracedays
[VERBOSE] maxdays=$tmpmaxdays
[VERBOSE] expmode = $expmode
[VERBOSE] noprojmode = $noprojmode
[VERBOSE] nopocmode = $nopocmode
[VERBOSE] badmode = $badmode

EOF

	print STDERR "[VERBOSE]: metafiles are :\n\t", (join "\n\t", @$metafiles), "\n";
}

my $tmp = read_metainfo_files($metafiles);
die "Error sorting metadata: $tmp\n" unless $tmp && ref($tmp) eq 'HASH';
my $metainfo = $tmp->{good};
my $badhash = $tmp->{bad};
die "Invalid good key ($metainfo), expecting list ref of metainfo at "
	unless $metainfo && ref($metainfo) eq 'ARRAY';
die "Invalid bad key ($badhash), expecting hash ref of bad data at "
	unless $badhash && ref($badhash) eq 'HASH';

my $options = 
{	now => $now,
	gracedays => $gracedays,
	maxdays => $maxdays,
};
	
my $sorted = categorize_metainfo_files($hpc, $metainfo, $options);
die "Error categorize metadata: $sorted\n" unless $sorted && ref($sorted) eq 'HASH';

my $badhash2 = $sorted->{bad};
my $noprojs = $sorted->{noprojects};
my $nopocs = $sorted->{nopocs};
my $expired = $sorted->{expired};
my $expbypoc = $sorted->{expbypoc};
my $bypoc = $sorted->{bypoc};

#Combine badhash hashes
$badhash = { %$badhash, %$badhash2 };

#First, send email to PoCs for standard (and maybe expired) archives

#Get list of pocs to email
my @poclist = keys %$bypoc;
if ( $expmode eq 'POC' )
{	my @poc2 = keys %$expbypoc;
	push @poclist, @poc2;
	my %temp = map { $_ => undef } @poclist;
	@poclist = keys %temp;
}

my $tmplate_file = "${cluster}_poc_warn_impending_data_deletion.tmplate";

foreach my $pocname (@poclist)
{	my $poc = Glue::HPCC::Project::Person->person_by_username(
		$hpc, $pocname);
	die "Error looking up info for PoC $pocname: $poc\n"
		unless $poc && ref($poc);

	my $pocemail = $poc->email;
	my $pocfname = $poc->fullname;

	my $mdata = $bypoc->{$pocname} || [];
	if ( $expmode eq 'POC' )
	{	my $tmp = $expbypoc->{$pocname} || [];
		push @$mdata, @$tmp;
	}

	my $poc_sorted = sort_metadata_for_poc($hpc, $pocname, $mdata);

	send_template_email($tmplate_file, 
	    {	Cluster => $cluster,
		PocUname => $pocname,
		PocEmail => $pocemail,
		PocFname => $pocfname,
		DeletionsList => $poc_sorted,
	    },
	);
}

#Generate system staff email

#This is a list of strings, one per line.  If string starts
#with a #, it is considered the start of a <ul>
my @sysmail = ();

#Archives w/out projects
if ( $noprojs && ref($noprojs) eq 'ARRAY' && @$noprojs )
{	#We have archives w/out any associated projects
	
	if ( $noprojmode eq 'DELETE' )
	{	push @sysmail, '#Deleting archives w/out associated projects:';
		foreach my $rec (@$noprojs)
		{	my $mifile = $rec->{MetadataSource};
			my $path = $rec->{DirectoryArchived};
			my $type = $rec->{DirectoryType};

			my $res;
			if ( $type eq 'home' )
			{	$res = delete_archived_home_dir(
					archive => $mifile, 
					hpc => $hpc,
					gracedays => $gracedays,
				);
			} else
			{	$res = delete_archived_data_dir(
					archive => $mifile, 
					hpc => $hpc,
					gracedays => $gracedays,
				);
			};
			$res = "deleted" unless $res;

			push @sysmail, "$mifile ($path): $res";
		}
	} elsif ( $noprojmode eq 'EMAIL' )
	{	push @sysmail, "#The following archives have no associated projects:";
		foreach my $rec (@$noprojs)
		{	my $mifile = $rec->{MetadataSource};
			my $path = $rec->{DirectoryArchived};
			my $type = $rec->{DirectoryType};

			push @sysmail, "$mifile ($path)";
		}
	} elsif ( $noprojmode eq 'IGNORE' )
	{	#Nothing to do
	} else
	{	die "Invalid value $noprojmode for noprojects-mode at ";
	}
}

#Archives w/out PoCs
if ( $nopocs && ref($nopocs) eq 'ARRAY' && @$nopocs )
{	#We have archives w/out any non-excluded PoCs
	
	if ( $nopocmode eq 'DELETE' )
	{	push @sysmail, '#Deleting archives w/out any non-excluded PoCs:';
		foreach my $rec (@$nopocs)
		{	my $mifile = $rec->{MetadataSource};
			my $path = $rec->{DirectoryArchived};
			my $type = $rec->{DirectoryType};

			my $res;
			if ( $type eq 'home' )
			{	$res = delete_archived_home_dir(
					archive => $mifile, 
					hpc => $hpc,
					gracedays => $gracedays,
				);
			} else
			{	$res = delete_archived_data_dir(
					archive => $mifile, 
					hpc => $hpc,
					gracedays => $gracedays,
				);
			};
			$res = "deleted" unless $res;

			push @sysmail, "$mifile ($path): $res";
		}
	} elsif ( $nopocmode eq 'EMAIL' )
	{	push @sysmail, "#The following archives have no non-excluded PoCs:";
		foreach my $rec (@$nopocs)
		{	my $mifile = $rec->{MetadataSource};
			my $path = $rec->{DirectoryArchived};
			my $type = $rec->{DirectoryType};
			my $excl = $rec->{ExcludePoCs} || [];
			$excl = join ", ", @$excl;

			push @sysmail, "$mifile ($path) [Excluded PoCs=$excl]";
		}
	} elsif ( $nopocmode eq 'IGNORE' )
	{	#Nothing to do
	} else
	{	die "Invalid value $nopocmode for nopocs-mode at ";
	}
}

#Expired Archives
if ( $expired && ref($expired) eq 'ARRAY' && @$expired )
{	#We have expired archives (expiration more than gracedays in past)
	
	if ( $expmode eq 'DELETE' )
	{	push @sysmail, "#Deleting expired archives (expired more than $gracedays days ago)";
		foreach my $rec (@$expired)
		{	my $mifile = $rec->{MetadataSource};
			my $path = $rec->{DirectoryArchived};
			my $type = $rec->{DirectoryType};
			my $expdate = $rec->{Expiration};

			my $res;
			if ( $type eq 'home' )
			{	$res = delete_archived_home_dir(
					archive => $mifile, 
					hpc => $hpc,
					gracedays => $gracedays,
				);
			} else
			{	$res = delete_archived_data_dir(
					archive => $mifile, 
					hpc => $hpc,
					gracedays => $gracedays,
				);
			};
			$res = "deleted" unless $res;

			push @sysmail, "$mifile ($path) [$expdate]: $res";
		}
	} elsif ( $expmode eq 'EMAIL' )
	{	push @sysmail, "#The following archives have expired more than $gracedays days ago:";
		foreach my $rec (@$expired)
		{	my $mifile = $rec->{MetadataSource};
			my $path = $rec->{DirectoryArchived};
			my $type = $rec->{DirectoryType};
			my $expdate = $rec->{Expiration};

			push @sysmail, "$mifile ($path) [$expdate]";
		}
	} elsif ( $expmode eq 'POC' )
	{	#Don't email if this is the only thing in email
		if ( @sysmail)
		{	push @sysmail, "#Exp-Mode = $expmode";
			my $tmp = scalar(@$expired);
			push @sysmail, "$tmp expired archives included in email to PoCs";
		}
	} elsif ( $expmode eq 'IGNORE' )
	{	#Nothing to do
	} else
	{	die "Invalid value $expmode for exp-mode at ";
	}
}

#Archives w bad metadata
if ( $badhash && ref($badhash) eq 'HASH' && %$badhash )
{	#We have archives w bad metadata
	
	if ( $badmode eq 'EMAIL' )
	{	push @sysmail, "#The following archives have bad metadata:";
		foreach my $key (keys %$badhash)
		{	my $val = $badhash->{$key};
			push @sysmail, "$key: $val";
		}
	} elsif ( $badmode eq 'IGNORE' )
	{	#Nothing to do
	} else
	{	die "Invalid value $badmode for bad-mode at ";
	}
}
		
$tmplate_file = "${cluster}_sysadmin_archive_cleanup.tmplate";
if ( @sysmail )
{	send_template_email($tmplate_file, 
	    {	Cluster => $cluster,
	    	Text => \@sysmail,
	    },
	);
}




