#!/usr/local/bin/perl
#Show job statistic for an allocation account
#usage() below globals section

use strict;
use warnings;

use lib '/cell_root/software/HPCC/lib/site_perl';
use Slurm::Sacct;

use Getopt::Long;
use Time::ParseDate;
use Time::Local qw(timelocal_nocheck);
use POSIX qw(strftime floor);

#---------------------------------------------------------------
#		Globals
#---------------------------------------------------------------

my $Sacct = "Slurm::Sacct";
#$Sacct->verbose(1);

#This is the maximum walltime allowed for jobs being charged, in seconds
my $MAX_WALLSECS_FOR_JOB = 2*7*24*60*60;

my $BEGINNING_OF_TIME = '1990-01-01'; #Earliest date, precedes all clusters

#For some queries, sacct command seems to time out.  Not good.
#Seems to be if a large amount of data being returned
#E.g., when long time period and/or large number of allocations
#$sacct_split_allocs causes us to split sacct queries by allocations; 
#if 0, run for all allocs specified in single command.
#if non-zero, max number of allocations per query.
my $sacct_split_allocs=1;
#Should we have a $sacct_split_time to split the time period ---
#issue, how do we avoid duplicate job entries???

#List of known output fields, for --format= option
#List of hash refs, with keys
#	name= Name of field (case as want displayed in help text)
#	title What displayed in header.  Use name if omitted
#	desc= Description (help text)
#	nonnumeric= If set, this is a string/non-numeric field
#	width=  Default width
#	decs=   Default decimal places.  -1 means no dec point
#	byuser= if set, requires byuser or byjob to be set
#	byjob=  if set, requires byjob to be set
#	debug=  if set, don't display in help text
my @output_fields =
(	{	name => 'Cluster',
		desc => 'the name of the cluster',
		nonnumeric => 1,
		width => 8,
	},

	{	name => 'Account',
		desc => 'the name of the allocation account',
		nonnumeric => 1,
		width => 20,
	},

	{	name => 'JobCount',
		desc => 'the number of jobs for the allocation account during time considered',
		width => 9,
		decs => -1,
	},

	{	name => 'FirstJob',
		title => '1stJob',
		desc => 'the jobid of first job for this allocation',
		width => 9,
		nonnumeric => 1, #Job arrays are possible, so use string
		decs => -1,
	},

	{	name => 'FirstJobDate',
		title => 'Date 1stJob',
		nonnumeric => 1,
		desc => 'the start time of first job for allocation',
		width => 12,
		decs => -1,
	},

	{	name => 'LastJob',
		desc => 'the jobid of last job for this allocation',
		width => 9,
		decs => -1,
		nonnumeric => 1, #Job arrays are possible, so use string
		decs => -1,
	},

	{	name => 'LastJobDate',
		desc => 'the end time of last job for allocation',
		nonnumeric => 1,
		width => 12,
		decs => -1,
	},

	{	name => 'ReqCPUs',
		title => 'ReqCPU',
		desc => 'the total number of CPU-cores _requested_ by all jobs  for this allocation account',
		width => 7,
		decs => -1,
	},

	{	name => 'CPUs',
		desc => 'the total number of CPU-cores _allocated_ by all jobs for allocation account.',
		width => 7,
		decs => -1,
	},

	{	name => 'UsageKSU',
		title => 'Usg kSU',
		desc => 'usage for the job(s) in this record during specified time period, in kSU ( 1 kSU=1000 SU)',
		width => 8,
		decs =>  1,
	},

	{	name => 'UsageSU',
		title => 'Usg SU',
		desc => 'usage for the job(s) in this record during specified time period, in SU (1 SU=1 CPU-core-hour = 60 cpu-min )',
		width => 8,
		decs =>  -1,
	},

	{	name => 'UsageMin',
		title => 'Usg min',
		desc => 'usage for the job(s) in this record during specified time period, in cpu-min ( 1 cpu-min = 1 CPU-core-min = 60 cpu-sec)',
		width => 8,
		decs =>  -1,
	},

	{	name => 'UsageSec',
		title => 'Usg sec',
		desc => 'usage for the job(s) in this record during specified time period, in cpu-sec (  1 CPU-core-sec)',
		width => 8,
		decs =>  -1,
	},

	{	name => 'Usage',
		desc => 'usage for job(s) in this record during specified time period, in unit specified by --unit flag.  See UsageKSU, UsageSU, etc. for details',
	},

	{	name => 'FullUsageKSU',
		title => 'FUsg kSU',
		desc => 'usage for the job(s) in this record for life of the job, in kSU ( 1 kSU=1000 SU)',
		width => 9,
		decs =>  3,
	},

	{	name => 'FullUsageSU',
		title => 'FUsg SU',
		desc => 'usage for the job(s) in this record for life of the job, in SU (1 SU=1 CPU-core-hour = 60 cpu-min )',
		width => 9,
		decs =>  1,
	},

	{	name => 'FullUsageMin',
		title => 'FUsg min',
		desc => 'usage for the job(s) in this record for life of the job, in cpu-min ( 1 cpu-min = 1 CPU-core-min = 60 cpu-sec)',
		width => 9,
		decs =>  -1,
	},

	{	name => 'FullUsageSec',
		title => 'FUsg sec',
		desc => 'usage for the job(s) in this record for life of the job, in cpu-sec (  1 CPU-core-sec)',
		width => 9,
		decs =>  -1,
	},

	{	name => 'FullUsage',
		desc => 'usage for job(s) in this record for life of the job, in unit specified by --unit flag.  See UsageKSU, UsageSU, etc. for details',
	},

);

#Generate hash keys by case insensitive name
my %output_fields = map { (lc $_->{name}) => $_ } @output_fields;

my @default_fields_std = ( 'Account', 'JobCount', 'CPUs', 'Usage', 'FirstJob', 'FirstJobDate', 'LastJob', 'LastJobDate' );


#Multiplier to convert given unit to cpu-sec
my %unit_conversion_to_cpusec =
(	'cpu-sec' => 1,
	'cpu-min' => 60,
	'su' => 60*60,
	'ksu' => 1000*60*60,
);

#---------------------------------------------------------------
#		Usage information
#---------------------------------------------------------------

sub usage_general(;$)
{	my $debug = shift;

	my $defflds_std = join ",", @default_fields_std;

	print STDERR <<EOF;
$0: Display usage information for Slurm account(s)

Usage:

$0 [ --account=ACCOUNT ] [ --account=ACCOUNT2,ACCOUNT3 ]  \
   [ --file=ACCOUNT_FILE ] \
   [ --treat-as-projects ] \
   [ --cluster=CLUSTER ] \
   [ --start=STARTDATE ] [ --end=ENDDATE ] \
   [ --unit=(kSU|SU|cpu-min|cpu-sec) ] \
   [ --machine-parsable ] [ --[no]headers ] \
   [ --[no]suppress0 ] \
   [ --combine-project-allocations ] \
   [ --fields=FLD1,FLD2,FLD3,... ]

ACCOUNT specifies the Slurm allocation account(s) to display the usage
information for.  You may specify multiple accounts, either by giving a 
comma delimited string of account names, and/or by repeating the --account 
parameter.  You may also specify an ACCOUNT_FILE which lists accounts, one
per line.  You can mix --account and --file arguments as well.  But at 
least one account must be specified in some fashion.  If --treat-as-projects
flag is given, any accounts specified (either with --account flag or in
the ACCOUNT_FILE) will be treated as project names, and we will treat as
meaning both ACCOUNT and ACCOUNT-hi (for projects without a high priority
allocation account, the ACCOUNT-hi will just be silently ignored).

CLUSTER is the cluster to request usage information about.  It defaults
based on the machine you are running the command on.

STARTDATE and ENDDATE specify the start and end dates for the report.
A variety of formats will be accepted, but the preferred are 
YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS.  If ENDDATE is omitted, all jobs after
STARTDATE are considered.  If STARTDATE is omitted, all jobs before ENDDATE
are considered.  If both omitted, all jobs are considered.

--unit allows one to specify what units the usage should be displayed in.
Options are kSU (1000 SU), SU (1 CPU-core-hour), cpu-min
(1 CPU-core-min) or cpu-sec (1 CPU-core-sec).  The default is 'kSU'.

If --machine-parsable is set (the default is --nomachine-parsable), the
output will be presented in a machine parsable fashion, with records separated
by the pipe (|) character.  If --noheaders is given (default is --headers),
the initial header line will NOT be printed.

Normally, the script will present results for each allocation account
named.  In particular, for projects with both high and standard priority
allocation accounts, there will be separate statistics collected for the
two priorities.  The flag --combine-project-allocations will cause the 
statistics to be merged for the two priorities of the allocation (under
the name of the standard allocation account, i.e. the project name).
By default, accounts for which no jobs have been found will not appear in
the output.  Use --nosuppress0 to have these displayed (with all zeros).
[NOTE: if --nosuppress0 is used with --treat-as-projects, you may receive
bogus entries for non-existant ACCOUNT-hi allocation accounts.]

If --fields is given, you can specify exactly which fields should be displayed
and in what order.  Give the '--help fields' for more information.  The default is:
	--fields=$defflds_std
EOF

}

sub _usage_print_field($;$)
#Helper for usage_fields
#Prints the usage text for an individual field name
{	my $fldname = lc shift;
	my $debug = shift;

	my $fld = $output_fields{$fldname};
	die "Invalid field name $fldname at " 
		unless $fld && ref($fld) eq 'HASH';

	my $name = $fld->{name};
	my $desc = $fld->{desc};
	my $width = $fld->{width};
	my $decs = $fld->{decs};
	my $dbg = $fld->{debug};

	return if $dbg && ! $debug;
	$desc .= " (DEBUG)" if $dbg;

	my $fmtspec = '';
	if ( defined $width )
	{	$fmtspec = ':' . $width;
		if ( defined $decs && $decs >= 0 )
		{	$fmtspec .= '.' . $decs;
		}
	}

	print STDERR "\t${name}${fmtspec} - $desc\n";
}

sub usage_fields(;$)
{	my $debug = shift;
	print STDERR <<EOF;

The --fields parameter allows you to specify exactly which fields should
be output, in what order, and formatting options for each field.  The value
given to the parameter is a comma separated list of "field specifiers".  

A "field specifier" consists of a case insensitive "field name" optionally
followed by a "format specifier".  Valid field names (along with their
default format specifier) are listed below.

A "format specifier" consists of a colon (:) either followed by a single
integer or two integers separated by a dot (.).  The first integer specifies
the minimum width for the field, and must be positive.  The second integer, if 
present, gives the the number of decimal places to print.  If 0, a decimal 
point will be printed and nothing after it, if negative no decimal point.  The 
second integer is only used for numeric data types, and defaults to -1.  The 
first integer is ignored when --machine-parsable is specified.

The valid fields are:
EOF

	my @flds = grep { ! ( $_->{byuser} || $_->{byjob} ) } @output_fields;
	@flds = map { lc ($_->{name})  } @flds;
	@flds = sort @flds;

	foreach my $fname (@flds)
	{	_usage_print_field($fname);
	}
		
	print STDERR <<EOF;

The difference between Usage and FullUsage is subtle.  For jobs that start after the
the start time of the report and end before the end time of the report, there is no difference.
For jobs whose run-time straddles the start time and/or end time of the report, "FullUsage"
is the number of SUs used by the job over its full lifetime.  "Usage" is the portion of "FullUsage"
that was consumed between the start and end time of the report.  E.g., if a job ran for 4 days, but
started exactly 3 days before the start time of the report (and the report is for at least 1 day),
"Usage" will only be 25% of "FullUsage".

EOF
		
}

sub get_accounts_from_file($)
#Returns a list ref of account names obtained from the file
#Account names should be listed one per line
#Blank lines and anything after a # are ignored
{	my $fname = shift;
	my $FILE;
	open($FILE, "<$fname") or 
		die "Unable to open account file $fname for reading: $!";

	my $accounts = [];
	LINE: while ( my $line = <$FILE> )
	{	chomp $line;
		next LINE unless $line;
		#Remove comments
		$line =~ s/#.*$//;
		#Remove leading/trailing whitespace
		$line =~ s/^\s*//; $line =~ s/\s*$//;
		#Skip empty lines
		next LINE unless $line;

		#We got an account
		push @$accounts, $line;
	}
	return $accounts;
}

#---------------------------------------------------------------
#		Do the real work
#---------------------------------------------------------------

sub process_job_record(@)
#Given a jobrecord (i.e., one line/record from a sacct list output)
#and a start and end timestamp (secs from 1970), process the record.
#First, check if the job actually ran during the period in question;
#if not return undef.
#Otherwise, returns a hash ref as described in get_job_data.
#In particular, calculate rawusage_tper.
{	my %arguments = @_;
	my $me = 'process_job_record';
	my $jobrec = delete $arguments{jobrecord};
	my $startts = delete $arguments{startts};
	my $endts = delete $arguments{endts};
	my $now = delete $arguments{now}; #Current ts, for use if end undef

	die "$me: Missing required parm 'jobrecord' at " unless $jobrec;
	#die "$me: Missing required parm 'startts' at " unless $startts;
	#die "$me: Missing required parm 'endts' at " unless $endts;
	if ( %arguments )
	{	my @tmp = keys %arguments;
		my $tmp = join ", ", @tmp;
		die "$me: Extraneous arguments [ $tmp ], aborting at ";
	}
	die "$me: Invalid jobrecord '$jobrec', expecting hashref at "
		unless $jobrec && ref($jobrec);
	#die "$me: Invalid start/end timestamps; start=$startts > end=$endts at "
	#	if $startts > $endts;

	$now = time unless defined $now;

	my $jstart = $jobrec->Start;
	my $jend   = $jobrec->End;
	my $jelapsed = $jobrec->Elapsed;
	my $jobid  = $jobrec->JobID;
	my $jobname = $jobrec->JobName;
	my $cluster = $jobrec->Cluster;
	my $account = $jobrec->Account;
	my $user    = $jobrec->User;
	my $state   = $jobrec->State;
	my $part    = $jobrec->Partition;
	my $reqcpus = $jobrec->ReqCPUS;
	my $cpus    = $jobrec->AllocCPUS;
	my $usage   = $jobrec->CPUTimeRAW;
        #Dropping energy stuff; only reported in job steps which we skip
	#my $fullenergy = $jobrec->ConsumedEnergy || 0;

	#print STDERR "Looking at job $jobid ...\n";

	#Convert jstart, jend to timestamps
	#jend will default to $now
	return unless $jstart; #Job has not started yet
	return if $jstart eq 'Unknown' ; #Job has not started yet
	my $jsts = parsedate($jstart,
		WHOLE=>1,
		VALIDATE=>1,
		PREFER_PAST=>1,
	);
	my $jets = $now;
	if ( $jend && $jend ne 'Unknown' )
	{	$jets = parsedate($jend,
			WHOLE=>1,
			VALIDATE=>1,
			PREFER_PAST=>1,
		);
	}

	return if defined $endts && $jsts > $endts; #Job started after our window
	return if defined $startts && $jets < $startts; #Job ended before our window

	my $job_secs = $jets - $jsts;
	return unless $job_secs > 0; #Skip 0 sec jobs
	
	my $jsts_tper = $jsts;
	$jsts_tper = $startts if defined $startts && $startts > $jsts;
	my $jets_tper = $jets;
	$jets_tper = $endts if defined $endts && $endts < $jets;
	my $job_secs_tper = $jets_tper - $jsts_tper;

	#Calculate what part of usage was in our window
	my $usage_tper;
	#my $tperenergy;
	if ( $job_secs_tper == $job_secs )
	{	#Job fits completely in our window
		$usage_tper = $usage;
		#$tperenergy = $fullenergy;
	} else
	{	
		my $frac = $job_secs_tper/$job_secs;
		$usage_tper = floor(0.5 + $usage * $frac);
		#This might not be correct, but is our best estimate
		#$tperenergy = floor(0.5 + $fullenergy * $frac); 
	}

	my $job = 
	{	jobid => $jobid,
		jobname => $jobname,
		cluster => $cluster,
		account => $account,
		user => $user,
		state => $state,
		partition => $part,
		reqcpus => $reqcpus,
		cpus => $cpus,
		jobcount => 1,
		rawusage => $usage,
		effrawusage => $usage_tper,
		start => $jstart,
		end => $jend,
		elapsed => $jelapsed,
		startts => $jsts,
		endts => $jets,
		elapsedsecs => $job_secs,
		effstartts => $jsts_tper,
		effendts => $jets_tper,
		effelapsedsecs => $job_secs_tper,
		#fullenergy => $fullenergy,
		#tperenergy => $tperenergy,
	};

	return $job;
}

sub get_job_data(@)
#Routine to collect data on all relevant jobs
#Takes key => value pairs
#	cluster: name of cluster, required
#	accounts: list ref of accounts to include, required
#	startts: timestamp (secs from 1970) of start of time window, required
#	endts: timestamp of end of time window, required
#	split_accounts: if 0, request all accounts in the sacct command.
#		if > 0, then limit to at most that many accounts per sacct command,
#		running multiple sacct commands to get the needed data.
#
#We want all jobs for cluster/accounts with any overlap in their runtime
#with startts/endts.  We use $MAX_WALLSECS_FOR_JOB and look for all jobs
#that started after startts-$MAX_WALLSECS_FOR_JOB, and ended  before
#endts+$MAX_WALLSECS_FOR_JOB (or are still running if current time is less
#then endts+$MAX_WALLSECS_FOR_JOB).  Those are the times given to sacct cmd,
#we then trim out jobs not overlapping our time window.
#
#We return a list ref of hash refs, with each hash containing then fields:
#	jobid: jobid of the job
#	jobname: name of the job
#	user: username of owner of the job
#	cluster: name of cluster for job
#	account: name of account for job
#	state: state of the job
#	partition: partition for the job
#	reqcpus: # of CPUs requested for job
#	cpus: # of cpus allocated to job
#	jobcount: set to 1
#	rawusage: full usage of job in cpu-sec
#	effrawusage: effective usage of job in cpu-sec (ie usage during this time-period)
#	start, end: start and end of job (human-readable dates)
#	elapsed: elapsed time of entire job, to-date (human-readable)
#	startts, endts: timestamp of start/end of job (seconds from 1970)
#	elapsedsecs: elapsed time of entire job, in seconds
#	effstartts, effendts: timestamp of start/end of job intersected with timeperiod
#	effelapsedsecs: elapsed time of job overlapping this timeperiod
#	#NOT DONE: fullenergy: energy consumption of life of job, to date
#	#NOT DONE: tperenergy: energy consumption of job during this timeperiod (estimated)
#
#rawusage_full is the full usage of a completed job over its lifetime
#(or the to-date usage of a running job).  
#rawusage_tper is the portion of rawusage_full that was consumed btwn
#startts and endts (NO adjustments for max walltime in those limits).
#If the job started and completed within [startts, endts], the two
#are the same.  Otherwise, we compute what fraction of the total runtime
#occured within [startts, endts] and adjust the usage correspondingly.
#	
{	my %arguments = @_;
	my $me = 'get_job_data';
	my $cluster = delete $arguments{cluster};
	my $accounts = delete $arguments{accounts};
	my $startts = delete $arguments{startts};
	my $endts = delete $arguments{endts};
	my $maxaccts = delete $arguments{split_accounts} || 0;

	die "$me: Missing required parm 'cluster' at " unless $cluster;
	die "$me: Missing required parm 'accounts' at " unless $accounts;
	#die "$me: Missing required parm 'startts' at " unless $startts;
	#die "$me: Missing required parm 'endts' at " unless $endts;
	if ( %arguments)
	{	my @tmp = keys %arguments;
		my $tmp = join ", ", @tmp;
		die "$me: Extraneous arguments [ $tmp ], aborting at ";
	}
	$accounts = [ $accounts ] unless ref($accounts);
	die "$me: Invalid value '$accounts' for 'accounts', expecting array ref, at "
		unless $accounts && ref($accounts) eq 'ARRAY';
	die "$me: Empty array given for 'accounts' at " unless @$accounts;
	die "$me: Invalid value $maxaccts < 0 given for maxaccts, at " if $maxaccts < 0;

	my ( $sts, $stime, $ets, $etime);
	if ( defined $startts )
	{	$sts = $startts - $MAX_WALLSECS_FOR_JOB;
		$stime = strftime("%Y-%m-%dT%H:%M:%S", localtime($sts));
	} else
	{	$stime = $BEGINNING_OF_TIME;
		$startts =  parsedate($stime,
			WHOLE=>1,
			VALIDATE=>1,
			PREFER_PAST=>1,
		);
		$sts = $startts;
	}

	if ( defined $endts )
	{	$ets = $endts + $MAX_WALLSECS_FOR_JOB;
		$etime = strftime("%Y-%m-%dT%H:%M:%S", localtime($ets));
	}
	
	my @accounts = @$accounts;
	my @jrecords = ();
	#my @sacct_args = ( "--clusters=$cluster", "--accounts=$acctlist", '-X' );
	my @sacct_args = ( "--clusters=$cluster", '-X' );
	push @sacct_args, '-S' => $stime if defined $stime; 
	push @sacct_args, '-E' => $etime if defined $etime; 

	while (@accounts)
	{	my @tmpaccounts = (); #Accounts we use for this sacct run
		if ($maxaccts > 0 && scalar(@accounts) > $maxaccts )
		{	#Restrict to maxaccts accounts this sacct call
			@tmpaccounts = splice(@accounts,0,$maxaccts);
		} else
		{	#Either maxaccts==0, or we have only have fewer than $maxaccts anyway
			@tmpaccounts = @accounts;
			@accounts = ();
		}
		my $acctlist = join ",", @tmpaccounts;


		my $jrecords = $Sacct->sacct_list(@sacct_args, "--accounts=$acctlist");
		unless ( $jrecords && ref($jrecords) eq 'ARRAY' )
		{	die "$me: Error running sacct cmd: $jrecords, aborting at ";
		}

		#Add these records to our list
		push @jrecords, @$jrecords;
	}

	my @jobs = ();
	foreach my $jrec (@jrecords)
	{	my $job = process_job_record( 
			jobrecord => $jrec,
			startts => $startts,
			endts => $endts,
		);
		push @jobs, $job if $job;
	}

	return [ @jobs ];
}

#These specify what fields get 'merged' and how
#	These fields copied from new to summary if not present in summary
my @_merge_copy_fields = qw(cluster account);
#	These are extra copy fields for when keep_user_info is set
my @_merge_copy_fields_user = qw(user);
#	These fields get added together
my @_merge_add_fields = 
(	'jobcount',
	'rawusage',
	'effrawusage',
	'reqcpus', #???
	'cpus', #??? 
	'elapsedsecs', #???
	'effelapsedsecs', #???
	#'tperenergy', #? energy only reported in job steps, which we skip
	#'fullenergy', #????
);

sub merge_job_records($$@)
#Takes a job summary hash (possibly undef) and a job hash
#(e.g. an element of output of get_job_data)
#and merges the records.
#Additional flags specified in options control the process,
#	keep_user_info: the user related fields (e.g. user) will be
#		preserved, normally are not.
#
#Returns the (new) job summary hash ref
{	my $job_summary = shift || {};
	my $newjob = shift;
	my %options = @_;
	my $me = 'merge_job_records';
	my $keep_user_info = delete $options{keep_user_info};

	die "$me: Missing job_summary at " unless $job_summary;
	die "$me: Missing newjob at " unless $newjob;
	die "$me: Invalid value '$job_summary' for 'job_summary', " .
		"expecting hash ref, at "
		unless $job_summary && ref($job_summary) eq 'HASH';
	die "$me: Invalid value '$newjob' for 'newjob', expecting hash ref, at "
		unless $newjob && ref($newjob) eq 'HASH';

	if (%options)
	{	my @tmp = keys %options;
		my $tmp = join ', ', @tmp;
		die "$me: Unrecognized options ($tmp), aborting at ";
	}

	my @copy_fields = @_merge_copy_fields;
	push @copy_fields, @_merge_copy_fields_user if $keep_user_info;
	my $fldname;
	COPY_FIELDS: foreach $fldname (@copy_fields)
	{	next COPY_FIELDS 
			if exists $job_summary->{$fldname} && $job_summary->{$fldname};
		$job_summary->{$fldname} = $newjob->{$fldname} if exists $newjob->{$fldname};
	}

	my @add_fields = @_merge_add_fields;
	ADD_FIELDS: foreach $fldname (@add_fields)
	{	my $old = $job_summary->{$fldname} || 0;
		my $new = $newjob->{$fldname} || 0;
		$job_summary->{$fldname} = $old + $new;
	}

	#Special cases; first/last job info
	my ($old, $new);
	$old = $job_summary->{firstjobts};
	$new = $newjob->{startts};
	unless ( defined $old && $old <= $new )
	{	#old undefined, or more recent then new
		$job_summary->{firstjobts} = $new;
		$job_summary->{firstjobid} = $newjob->{jobid};
	}

	$old = $job_summary->{lastjobts};
	$new = $newjob->{endts};
	unless ( defined $old && $old >= $new )
	{	#old undefined or $new is later
		$job_summary->{lastjobts} = $new;
		$job_summary->{lastjobid} = $newjob->{jobid};
	}
	return $job_summary;
}

sub collect_jobs_by_account($;$)
#Given a list ref of job records ( e.g. the output of get_job_data)
#collect them by cluster and account.  Will be reordered alpha by
#accountname, then clustername.
#
#If $project_combine given, treat foo and foo-hi accounts the same
#(as foo project) in the collection process.
#
#Returns a list ref of job records
{	my $jobs = shift;
	my $project_combine = shift;
	my $me = 'collect_jobs_by_account';
	die "$me: Missing required parameter 'jobs' at " unless $jobs;
	die "$me: Invalid value '$jobs' for 'jobs', expecting list ref, at "
		unless $jobs && ref($jobs) eq 'ARRAY';

	my %byacct = ();

	#Collect records together
	foreach my $job (@$jobs)
	{	my $cluster = $job->{cluster};
		my $account = $job->{account};

		if ( $project_combine )
		{	if ( $account =~ /-hi$/ )
			{	$account =~ s/-hi$//;
				$job->{account} = $account;
			}
		}

		my $chash = $byacct{$account};
		unless ( $chash && ref($chash) eq 'HASH' )
		{	$chash = $byacct{$account} = {};
		}

		my $acctsum = $chash->{$cluster};
		$acctsum = merge_job_records($acctsum, $job, keep_user_info=>0);
		$chash->{$cluster} = $acctsum;
		$byacct{$account} = $chash;
	}

	#Generate output list ref
	my @accounts = keys %byacct;
	@accounts = sort( @accounts );
	my @jobs = ();

	foreach my $account (@accounts)
	{	my $chash = $byacct{$account};
		my @clusters = keys %$chash;
		@clusters = sort(@clusters);
		foreach my $cluster (@clusters)
		{	my $jobrec = $chash->{$cluster};
			push @jobs, $jobrec;
		}
	}

	return [ @jobs ];
}

#---------------------------------------------------------------
#		Output the results
#---------------------------------------------------------------

#The fields mentioned below have same name in fldspec as in jobrecord
my @fields_matching_keys = qw
(	jobid
	jobname
	account
	cluster
	user
	state
	partition
	reqcpus
	cpus
	jobcount
	start
	end
	elapsed
	startts
	endts
	elapsedsecs
	effstartts
	effendts
	effelapsedsecs
);
#	fullenergy - only reported in job steps
my %fields_matching_keys = map { $_ => undef } @fields_matching_keys;


sub extract_field_from_record($$;$)
#Extracts field from record.
#Takes a hash ref for record (e.g. an element of list ref from
#get_job_data, or one of the collect_jobs_* routines), and a 
#field specifier (only need name).  Optionally takes $unit
{	my $record = shift;
	my $fldspec = shift;
	my $unit = shift;
	my $me = "extract_field_from_record";

	die "$me: Missing parameter 'record' at " unless $record;
	die "$me: Missing parameter 'fldspec' at " unless $fldspec;
	die "$me: Invalid value '$record' for 'record', expecting ref, at " 
		unless ref($record);
	die "$me: Invalid value '$fldspec' for 'fldspec', expecting ref, at " 
		unless ref($fldspec) eq 'HASH';

	my $fname = lc $fldspec->{name};
	if ( $fname eq 'usage' || $fname eq 'fullusage' )
	{	#Replace with unit specific version
		die "$me: Cannot use '$fname' without specifying unit at "
			unless $unit;
		my $tmp = lc $unit;
		$tmp =~ s/^cpu-//; #Strip cpu- part
		$fname .= $tmp;
	}

	return $record->{$fname} if ( exists $fields_matching_keys{$fname} );

	if ( $fname =~ /^(usage|fullusage)/ )
	{	#Get the raw value in cpu-secs
		my $usage = $record->{rawusage};
		$usage = $record->{effrawusage} if $fname=~ /^usage/;

		#Figure out what unit is desired
		my $tmpunit = $fname;
		$tmpunit =~ s/^(usage|fullusage)//;

		#Convert to desired unit
		$usage = 0 unless defined $usage;
		return $usage if $tmpunit eq 'sec';
		return $usage/60 if $tmpunit eq 'min';
		return $usage/60/60 if $tmpunit eq 'su';
		return $usage/60/60/1000 if $tmpunit eq 'ksu';

		die "$me: Invalid fieldname '$fname' at ";
	}

	if ( $fname eq 'fullname' )
	{	my $user = $record->{user};
		return unless $user;
		return get_fullname_by_username($user);
	}

	my ($fld, $val);
	if ( $fname eq 'firstjob' )
	{	$val = $record->{firstjobid};
		return $val;
	}
	if ( $fname eq 'firstjobdate' )
	{	$val = $record->{firstjobts};
		return 'n/a' unless defined $val;
		$val = strftime("%Y-%m-%d", localtime($val));
		return $val;
	}
	if ( $fname eq 'lastjob' )
	{	$val = $record->{lastjobid};
		return $val;
	}
	if ( $fname eq 'lastjobdate' )
	{	$val = $record->{lastjobts};
		return 'n/a' unless defined $val;
		$val = strftime("%Y-%m-%d", localtime($val));
		return $val;
	}

#	my ($fld, $val);
#	if ( $fname eq 'energy' )
#	{	$fld = 'tperenergy';
#		$val = $record->{$fld};
#		return $val;
#	}

	die "$me: Unrecognized field name '$fname', aborting at ";
}
		
sub dump_record(@)
#Dumps a single record, for use in dump_data
#Takes key => value pairs, recognizing
#	record: record to dump, a hash ref, REQUIRED
#	fields: list ref of field specifier hashes, REQUIRED
#	machine_parsable: boolean, if true do machine parsable output
{	my %arguments = @_;
	my $me = 'dump_record';
	my $record = delete $arguments{record};
	my $fields = delete $arguments{fields};
	my $machparse = delete $arguments{machine_parsable};
	my $unit = delete $arguments{unit};

	die "$me: Missing required argument 'record' at " unless $record;
	die "$me: Missing required argument 'fields' at " unless $fields;
	if ( %arguments )
	{	my @tmp = keys %arguments;
		my $tmp = join ', ', @tmp;
		die "$me: Extraneous arguments [ $tmp ], aborting at ";
	}
	die "$me: Invalid value '$record' for 'record', expecting hash ref at "
		unless $record && ref($record) eq 'HASH';
	die "$me: Invalid value '$fields' for 'fields', expecting array ref at "
		unless $fields && ref($fields) eq 'ARRAY';

	my $first = 1;

	FLDSPEC: foreach my $fldspec (@$fields)
	{	my $width = $fldspec->{width} || 0;
		my $decs  = $fldspec->{decs};
		my $nonnum= $fldspec->{nonnumeric};
		$decs = -1 unless defined $decs;
		$width = 0 if $machparse;

		print "|" if $machparse && ! $first;
		$first = 0;

		my $value = extract_field_from_record($record, $fldspec, $unit);
		unless ( defined $value )
		{	#Undef is always handled as a string
			printf "%*s", $width, '';
			next FLDSPEC;
		}

		if ( $nonnum )
		{	printf "%*s", $width, $value;
			next FLDSPEC;
		}

		#Numeric values
		if ( $decs < 0 )
		{	#Integral, no dec point
			printf "%*d", $width, $value;
			next FLDSPEC;
		}

		#Floating point
		printf "%*.*f", $width, $decs, $value;
		next FLDSPEC;
	}
	print "\n";
}

sub dump_data(@)
#Dumps the data we collected
#Takes key => value pairs for input, recognizing
#	data: a list ref of records to dump.  REQUIRED
#		Any combining of records by user/account is assumed
#		to have already occured
#	fields: a list ref of field specifier hashes.  REQUIRED
#	startts: timestamp (in seconds from 1970) of start of time period.  REQUIRED
#	endts: timestamp of end of period.  REQUIRED unless timeperiod given and 
#		is not 'custom'
#	timeperiod: name for period of time.  REQUIRED unless endts given (in
#		which case must not be 'custom')
#	accounts: list of accounts being considered.  REQUIRED
#	cluster: cluster being considered. REQUIRED
#	machine_parsable: Boolean, if true make machine parsable output
#	headers: Boolean, if true output field headers
#	unit: unit flag, in case bare 'usage' passed in fields
#	nosuppress0: normally accounts for which no jobs found will be 
#		suppressed from output.  If this is set, do not suppress them.
#	project_combine: if set, then we are dumping for projects, not accounts.  Adjust
#		some header text, etc.
{	my %arguments = @_;
	my $me = 'dump_data';
	my $data = delete $arguments{data};
	my $fields = delete $arguments{fields};
	my $startts = delete $arguments{startts};
	my $endts = delete $arguments{endts};
	my $timeperiod = delete $arguments{timeperiod};
	my $accounts = delete $arguments{accounts};
	my $cluster = delete $arguments{cluster};
	my $machparse = delete $arguments{machine_parsable};
	my $print_headers = delete $arguments{headers};
	my $unit = delete $arguments{unit};
	my $nosuppress0 = delete $arguments{nosuppress0};
	my $projcomb = delete $arguments{project_combine};

	die "$me: Missing required argument 'data' at " unless $data;
	die "$me: Missing required argument 'fields' at " unless $fields;
	die "$me: Missing required argument 'accounts' at " unless $accounts;
	die "$me: Missing required argument 'cluster' at " unless $cluster;
	#die "$me: Missing required argument 'startts' at " unless $startts;
	if ( %arguments )
	{	my @tmp = keys %arguments;
		my $tmp = join ', ', @tmp;
		die "$me: Extraneous arguments [ $tmp ], aborting at ";
	}
	die "$me: Invalid value '$data' for 'data', expecting array ref at "
		unless $data && ref($data) eq 'ARRAY';
	die "$me: Invalid value '$fields' for 'fields', expecting array ref at "
		unless $fields && ref($fields) eq 'ARRAY';
	die "$me: Invalid value '$accounts' for 'accounts', expecting array ref at "
		unless $accounts && ref($accounts) eq 'ARRAY';
	die "$me: Illegal empty array for 'fields' at " unless scalar(@$fields);
	die "$me: Illegal empty array for 'accounts' at " unless scalar(@$accounts);

	my $header;

	if ( $projcomb )
	{	#Delete any accounts with -hi in them; everything should be by project name
		$accounts = [ grep { $_ !~ /-hi$/ } @$accounts ];
	}

	if ( $print_headers )
	{	#Print info re time period being displayed and accounts
		my $tper = 'time period';
		$tper = $timeperiod if $timeperiod && $timeperiod ne 'custom';

		my ($start, $end);
		if ( defined $startts )
		{	$start = strftime("%a %b %d %Y %H:%M:%S", localtime($startts));
		} else
		{	$start = '<start of records>';
		}
		if ( defined $endts )
		{	$end = strftime("%b %d %H:%M:%S", localtime($endts)) if defined $endts;
		}

		my $tpstr = "";
		if ( defined $startts && defined $endts )
		{	$tpstr = "from $start to $end";
		} elsif ( defined $startts )
		{	$tpstr = "from $start";
		} elsif ( defined $endts )
		{	$tpstr = "until $end";
		} else
		{	$tpstr = "no time constraint";
		}

		$header = "Job stats for timeperiod: $tpstr";
		$header = "#$header" if $machparse;
		print "$header\n";

		if ($projcomb )
		{    $header = "For projects: " . (join ", ", @$accounts) . " on cluster $cluster";
		} else
		{    $header = "For alloc accts: " . (join ", ", @$accounts) . " on cluster $cluster";
		}
		$header = "#$header" if $machparse;
		print "$header\n";
	}

	unless ( scalar(@$data) )
	{	$header = "No jobs found matching specified criteria.";
		$header = "#$header" if $machparse;
		print "$header\n" if $print_headers;
		return;
	}

	#Print field headers
	if ( $print_headers )
	{	print "#" if $machparse;
		my $first = 1;
		foreach my $fldspec (@$fields)
		{	my $width = $fldspec->{width};
			$width = 0 unless $width && $width > 0;
			my $name = $fldspec->{title} || $fldspec->{name};
			if ( $machparse )
			{	#Ignore widths with $machparse
				print "|" unless $first;
				print $name;
			} else
			{ 	printf "%*s", $width, $name;
			}
			$first = 0;
		}
		print "\n";
	}

	#Print the actual data

	ACCOUNT: foreach my $acct (@$accounts)
	{	my @records = grep { $_->{account} eq $acct } @$data;
		unless ( scalar(@records) )
		{	next ACCOUNT unless $nosuppress0;
			my $fakerec = {	account => $acct,
					cluster => $cluster,
				};
			@records = ( $fakerec );
			#print STDERR "Making null record for account $acct\n";
		}
		foreach my $record (@records)
		{	dump_record( record=>$record, fields=>$fields, 
				machine_parsable=>$machparse, unit=>$unit);
		}
	}
}
			


#---------------------------------------------------------------
#		Main
#---------------------------------------------------------------

my $help;
my $accounts = [];
my $acct_files = [];
my $startdate; # = 'yesterday midnight'; #let Time::ParseDate do the work
my ($cluster, $enddate, );
my $unit = 'kSU';
my $machparse = 0;
my $headers = 1;
my $fields;
my (@tmp, $tmp);
my $debug;
my $suppress0 = 1;
my $treat_as_projs = 0;
my $projcomb = 0;

my $res = GetOptions(
	'help=s' => \$help,
	'debug!' => \$debug,
	'suppress0!' => \$suppress0,
	'treat-as-projects|treat_as_projects!' => \$treat_as_projs,
	'combine-project-allocations|combine_project_allocations!' =>
		\$projcomb,

	'machine-parsable|machine_parsable!' => \$machparse,
	'headers!' => \$headers,

	'cluster=s' => \$cluster,
	'unit=s' => \$unit,
	'startdate=s' => \$startdate,
	'enddate=s' => \$enddate,
	'fields=s' => \$fields,

	'a|A|account|accounts|accts=s' => $accounts,
	'files|account-files|account_files|acctfiles|acct-files|acct_files=s' 
		=> $acct_files,

	#For defaulting accounts
);

unless ($res)
{	usage_general($debug);
	die "\nError parsing arguments.\n";
} 
	
if ( $help )
{	$help = lc $help;
	if ( $help eq 'fields' || $help eq 'format' )
	{	usage_fields($debug);
	} else
	{	usage_general($debug);
	}
	exit 0;
}

unless ( $cluster )
{	$cluster = 'dt' if $ENV{HOSTNAME} =~ /deepthought\.umd.edu$/;
	$cluster = 'dt2' if $ENV{HOSTNAME} =~ /deepthought2\.umd.edu$/;
	die "Unable to default cluster from hostname $ENV{HOST} at "
		unless $cluster;
}
die "Illegal cluster '$cluster'; expecting dt or dt2\n" 
	unless $cluster eq 'dt' || $cluster eq 'dt2';


if ( scalar(@$acct_files) )
{	#Append contents of each acctfile to accounts
	foreach my $afiles (@$acct_files)
	{	my @afiles = split /\s*,\s*/, $afiles;
		foreach my $afile (@afiles)
		{	$tmp = get_accounts_from_file($afile);
			push @$accounts, @$tmp;
		}
	}
}
unless ( scalar(@$accounts) )
{	usage_general;
	die "You must specify at least one account.\n";
}

#Split any CSV fields into separate elements
@tmp = ();
foreach $tmp (@$accounts)
{	#Strip leading/trailing spaces
	$tmp =~ s/^\s*//; $tmp =~ s/\s*$//;
	my @tmp2 = split /\s*,\s*/, $tmp;
	push @tmp, @tmp2;
}
$accounts = [ @tmp ];

#Handle --treat-as-projects
#Dont worry about duplicates, will de-dup later
if ( $treat_as_projs )
{	@tmp = ();
	ACCT: foreach $tmp (@$accounts)
	{	push @tmp, $tmp;
		my $extra = $tmp;
		if ( $extra =~ /-hi$/ )
		{	#Given hi priority alloc, add std???
			$extra =~ s/-hi$//;
			push @tmp, $extra;
			next ACCT;
		} else
		{	#Got std priority, add high priority
			$extra .= "-hi";
			push @tmp, $extra;
		}
	}
	$accounts = [ @tmp ];
}

#Remove duplicate accounts
my %tmp = map { $_ => undef } @$accounts;
@tmp = keys %tmp;
@tmp = sort @tmp;
$accounts = [ @tmp ];

unless ( exists $unit_conversion_to_cpusec{lc $unit} )
{	usage_general;
	@tmp = keys %unit_conversion_to_cpusec;
	$tmp = join ", ", @tmp;
	die "\nIllegal value for --unit, want one of ($tmp)\n";
}

my ($startts, $endts);

if ( $startdate )
{	$startts = parsedate($startdate,
		WHOLE=>1,
		VALIDATE=>1,
		PREFER_PAST=>1,
	);
	unless ( defined $startts )
	{	die "Invalid STARTDATE '$startdate', for best results use " .
			"YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS\n";
	}
} 

if ( $enddate )
{	$endts = parsedate($enddate,
		WHOLE=>1,
		VALIDATE=>1,
		PREFER_PAST=>1,
	);
	unless ( defined $endts )
	{	die "Invalid ENDDATE '$enddate', for best results use " .
			"YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS\n";
	}
} 

##Convert back to startdate, enddate, in format Sacct will like
#$startdate = strftime("%Y-%m-%dT%H:%M:%S", localtime($startts) );
#$enddate = strftime("%Y-%m-%dT%H:%M:%S", localtime($endts) );

#Process validate output fields statement:
my @ofields;
if ( $fields )
{	#Strip leading/trailing spaces
	$fields =~ s/^\s*//; $fields =~ s/\s*$//;
	$fields = $fields;
	@ofields = split /\s*,\s*/, $fields;
} else
{	@ofields = @default_fields_std;
}

#Generate a list of field specifiers ala @output_fields
my @fields = ();
foreach my $fldspec (@ofields)
{	
	my $fldname = lc $fldspec;
	$fldname =~ s/:.*$//; #Remove format specification

	my $fmtspec = $fldspec;
	my ($width, $decs);
	if ( $fmtspec =~ /:/ )
	{	$fmtspec =~ s/^.*://; #Remove field name
		($width, $decs) = split /\./, $fmtspec;
		$decs = -1 unless defined $decs;
	} else
	{	$fmtspec = undef;
		$width = undef;
		$decs = undef;
	}

	if ( $fldname eq 'usage'  || $fldname eq 'fullusage' )
	{	#Handle usage specially; replace with usage . unit
		$tmp = lc $unit;
		$tmp =~ s/^cpu-//; #Strip cpu- part
		$fldname .= $tmp;
	}

	my $fld = $output_fields{$fldname};
	unless ($fld)
	{	die "Invalid field specifier '$fldspec'; use --help fields for more information\n";
	}
	
	#Make a copy
	$fld = { %$fld };
	if ( defined $width )
	{	$fld->{width} = $width;
		$fld->{decs} = $decs if defined $decs;
	}

	push @fields, $fld;
}

#Handle debug mode
$Sacct->verbose($debug);

#Done validating/processing parameters; now do real work

my $jobs = get_job_data(cluster=>$cluster,
		accounts=>$accounts,
		startts => $startts,
		endts => $endts,
		split_accounts => $sacct_split_allocs,
	);

$jobs = collect_jobs_by_account($jobs, $projcomb);

dump_data(
	data => $jobs,
	fields => \@fields,
	startts => $startts,
	endts => $endts,	
	accounts => $accounts,
	cluster => $cluster,
	machine_parsable => $machparse,
	headers => $headers,
	nosuppress0 => ($suppress0?0:1),
	project_combine => $projcomb,
);


