org.apache.pig.tools.pigstats.spark

Class SparkJobStats

java.lang.Object

org.apache.pig.newplan.Operator

org.apache.pig.tools.pigstats.JobStats

org.apache.pig.tools.pigstats.spark.SparkJobStats

All Implemented Interfaces:

java.io.Serializable

Direct Known Subclasses:

Spark1JobStats, Spark2JobStats

public abstract class SparkJobStats
extends JobStats

See Also:

Serialized Form

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.pig.tools.pigstats.JobStats

JobStats.JobState

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.mapred.Counters	counters
static java.lang.String	FS_COUNTER_GROUP

Fields inherited from class org.apache.pig.tools.pigstats.JobStats

ALIAS, ALIAS_LOCATION, conf, FAILURE_HEADER, FEATURE, hdfsBytesRead, hdfsBytesWritten, inputs, outputs, state, SUCCESS_HEADER

Fields inherited from class org.apache.pig.newplan.Operator

annotations, hashPrime, location, name, plan

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	SparkJobStats(int jobId, PigStats.JobGraph plan, org.apache.hadoop.conf.Configuration conf)
protected	SparkJobStats(java.lang.String jobId, PigStats.JobGraph plan, org.apache.hadoop.conf.Configuration conf)

Method Summary

Modifier and Type	Method and Description
void	accept(PlanVisitor v) Accept a visitor at this node in the graph.
void	addInputStats(POLoad po, boolean success, boolean singleInput)
void	addOutputInfo(POStore poStore, boolean success, JobStatisticCollector jobStatisticCollector)
void	collectStats(JobStatisticCollector jobStatisticCollector)
protected abstract java.util.Map<java.lang.String,java.lang.Long>	combineTaskMetrics(java.util.Map<java.lang.String,java.util.List<TaskMetrics>> jobMetric)
long	getAvgMapTime()
long	getAvgREduceTime()
java.lang.String	getDisplayString()
org.apache.hadoop.mapred.Counters	getHadoopCounters()
java.lang.String	getJobId()
long	getMapInputRecords()
long	getMapOutputRecords()
long	getMaxMapTime()
long	getMaxReduceTime()
long	getMinMapTime()
long	getMinReduceTime()
java.util.Map<java.lang.String,java.lang.Long>	getMultiInputCounters()
java.util.Map<java.lang.String,java.lang.Long>	getMultiStoreCounters()
int	getNumberMaps()
int	getNumberReduces()
long	getProactiveSpillCountObjects()
long	getProactiveSpillCountRecs()
long	getReduceInputRecords()
long	getReduceOutputRecords()
long	getSMMSpillCount()
java.util.Map<java.lang.String,java.lang.Long>	getStats()
java.util.Map<java.lang.String,SparkCounter<java.util.Map<java.lang.String,java.lang.Long>>>	getWarningCounters()
void	initWarningCounters()
void	setAlias(SparkOperator sparkOperator)
void	setConf(org.apache.hadoop.conf.Configuration conf)

Methods inherited from class org.apache.pig.tools.pigstats.JobStats

calculateMedianValue, getAlias, getAliasLocation, getBytesWritten, getErrorMessage, getException, getFeature, getHdfsBytesRead, getHdfsBytesWritten, getInputs, getOutputs, getOutputSize, getRecordWrittern, getState, isEqual, isIndexer, isSampler, isSuccessful, setBackendException, setErrorMsg, setSuccessful

Methods inherited from class org.apache.pig.newplan.Operator

annotate, getAnnotation, getLocation, getName, getPlan, removeAnnotation, setLocation, setPlan

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

counters

protected org.apache.hadoop.mapred.Counters counters

FS_COUNTER_GROUP

public static java.lang.String FS_COUNTER_GROUP

Constructor Detail

SparkJobStats

protected SparkJobStats(int jobId,
                        PigStats.JobGraph plan,
                        org.apache.hadoop.conf.Configuration conf)

SparkJobStats

protected SparkJobStats(java.lang.String jobId,
                        PigStats.JobGraph plan,
                        org.apache.hadoop.conf.Configuration conf)

Method Detail

setConf

public void setConf(org.apache.hadoop.conf.Configuration conf)

Overrides:

setConf in class JobStats

addOutputInfo

public void addOutputInfo(POStore poStore,
                          boolean success,
                          JobStatisticCollector jobStatisticCollector)

addInputStats

public void addInputStats(POLoad po,
                          boolean success,
                          boolean singleInput)

collectStats

public void collectStats(JobStatisticCollector jobStatisticCollector)

combineTaskMetrics

protected abstract java.util.Map<java.lang.String,java.lang.Long> combineTaskMetrics(java.util.Map<java.lang.String,java.util.List<TaskMetrics>> jobMetric)

getStats

public java.util.Map<java.lang.String,java.lang.Long> getStats()

getJobId

public java.lang.String getJobId()

Specified by:

getJobId in class JobStats

accept

public void accept(PlanVisitor v)
            throws FrontendException

Description copied from class: Operator

Accept a visitor at this node in the graph.

Specified by:

accept in class JobStats

Parameters:

v - Visitor to accept.

Throws:

FrontendException

getDisplayString

public java.lang.String getDisplayString()

Specified by:

getDisplayString in class JobStats

getNumberMaps

public int getNumberMaps()

Specified by:

getNumberMaps in class JobStats

getNumberReduces

public int getNumberReduces()

Specified by:

getNumberReduces in class JobStats

getMaxMapTime

public long getMaxMapTime()

Specified by:

getMaxMapTime in class JobStats

getMinMapTime

public long getMinMapTime()

Specified by:

getMinMapTime in class JobStats

getAvgMapTime

public long getAvgMapTime()

Specified by:

getAvgMapTime in class JobStats

getMaxReduceTime

public long getMaxReduceTime()

Specified by:

getMaxReduceTime in class JobStats

getMinReduceTime

public long getMinReduceTime()

Specified by:

getMinReduceTime in class JobStats

getAvgREduceTime

public long getAvgREduceTime()

Specified by:

getAvgREduceTime in class JobStats

getMapInputRecords

public long getMapInputRecords()

Specified by:

getMapInputRecords in class JobStats

getMapOutputRecords

public long getMapOutputRecords()

Specified by:

getMapOutputRecords in class JobStats

getReduceInputRecords

public long getReduceInputRecords()

Specified by:

getReduceInputRecords in class JobStats

getReduceOutputRecords

public long getReduceOutputRecords()

Specified by:

getReduceOutputRecords in class JobStats

getSMMSpillCount

public long getSMMSpillCount()

Specified by:

getSMMSpillCount in class JobStats

getProactiveSpillCountObjects

public long getProactiveSpillCountObjects()

Specified by:

getProactiveSpillCountObjects in class JobStats

getProactiveSpillCountRecs

public long getProactiveSpillCountRecs()

Specified by:

getProactiveSpillCountRecs in class JobStats

getHadoopCounters

public org.apache.hadoop.mapred.Counters getHadoopCounters()

Specified by:

getHadoopCounters in class JobStats

getMultiStoreCounters

public java.util.Map<java.lang.String,java.lang.Long> getMultiStoreCounters()

Specified by:

getMultiStoreCounters in class JobStats

getMultiInputCounters

public java.util.Map<java.lang.String,java.lang.Long> getMultiInputCounters()

Specified by:

getMultiInputCounters in class JobStats

setAlias

public void setAlias(SparkOperator sparkOperator)

getWarningCounters

public java.util.Map<java.lang.String,SparkCounter<java.util.Map<java.lang.String,java.lang.Long>>> getWarningCounters()

initWarningCounters

public void initWarningCounters()