@InterfaceAudience.Public @InterfaceStability.Stable public class PigServer extends java.lang.Object
Modifier and Type | Class and Description |
---|---|
protected class |
PigServer.Graph |
Modifier and Type | Field and Description |
---|---|
protected java.util.Deque<PigServer.Graph> |
graphs |
protected org.apache.commons.logging.Log |
log |
protected PigContext |
pigContext |
static java.lang.String |
PRETTY_PRINT_SCHEMA_PROPERTY |
protected java.lang.String |
scope |
Constructor and Description |
---|
PigServer(ExecType execType) |
PigServer(ExecType execType,
org.apache.hadoop.conf.Configuration conf) |
PigServer(ExecType execType,
java.util.Properties properties) |
PigServer(PigContext context) |
PigServer(PigContext context,
boolean connect) |
PigServer(java.util.Properties properties) |
PigServer(java.lang.String execTypeString) |
PigServer(java.lang.String execTypeString,
java.util.Properties properties) |
Modifier and Type | Method and Description |
---|---|
void |
addPathToSkip(java.lang.String path)
Add a path to be skipped while automatically shipping binaries for
streaming.
|
long |
capacity()
Returns the unused byte capacity of an HDFS filesystem.
|
void |
debugOff()
Set the logging level to the default.
|
void |
debugOn()
Set the logging level to DEBUG.
|
boolean |
deleteFile(java.lang.String filename)
Delete a file.
|
void |
discardBatch()
Discards a batch of Pig commands.
|
Schema |
dumpSchema(java.lang.String alias)
Write the schema for an alias to System.out.
|
Schema |
dumpSchemaNested(java.lang.String alias,
java.lang.String nestedAlias)
Write the schema for a nestedAlias to System.out.
|
java.util.List<ExecJob> |
executeBatch()
Submits a batch of Pig commands for execution.
|
java.util.List<ExecJob> |
executeBatch(boolean parseAndBuild)
Submits a batch of Pig commands for execution.
|
boolean |
existsFile(java.lang.String filename)
Test whether a file exists.
|
void |
explain(java.lang.String alias,
java.io.PrintStream stream)
Provide information on how a pig query will be executed.
|
void |
explain(java.lang.String alias,
java.lang.String format,
boolean verbose,
boolean markAsExecute,
java.io.PrintStream lps,
java.io.PrintStream eps,
java.io.File dir,
java.lang.String suffix)
Provide information on how a pig query will be executed.
|
long |
fileSize(java.lang.String filename)
Returns the length of a file in bytes which exists in the HDFS (accounts for replication).
|
java.util.Map<java.lang.String,LogicalPlan> |
getAliases()
Return a map containing the logical plan associated with each alias.
|
java.util.Set<java.lang.String> |
getAliasKeySet()
Get the set of all current aliases.
|
protected PigServer.Graph |
getClonedGraph()
Creates a clone of the current DAG
|
PigServer.Graph |
getCurrentDAG()
Current DAG
|
java.util.Map<Operator,DataBag> |
getExamples(java.lang.String alias) |
java.lang.String |
getJobName() |
java.lang.String |
getJobPriority() |
protected java.util.List<ExecJob> |
getJobs(PigStats stats)
Retrieves a list of Job objects from the PigStats object
|
java.lang.String |
getLastRel() |
LogicalPlanData |
getLogicalPlanData()
Returns data associated with LogicalPlan.
|
PigContext |
getPigContext() |
boolean |
isBatchEmpty()
Returns whether there is anything to process in the current batch.
|
boolean |
isBatchOn()
Retrieve the current execution mode.
|
boolean |
isDebugOn() |
protected PigStats |
launchPlan(LogicalPlan lp,
java.lang.String jobName)
A common method for launching the jobs according to the logical plan
|
java.lang.String[] |
listPaths(java.lang.String dir)
List the contents of a directory.
|
boolean |
mkdirs(java.lang.String dirs)
Make a directory.
|
java.util.Iterator<Tuple> |
openIterator(java.lang.String id)
Executes a Pig Latin script up to and including indicated alias.
|
protected java.util.List<java.lang.String> |
paramMapToList(java.util.Map<java.lang.String,java.lang.String> params) |
void |
parseAndBuild()
This method parses the scripts and builds the LogicalPlan.
|
void |
printAliases()
Intended to be used by unit tests only.
|
void |
printHistory(boolean withNumbers) |
void |
registerCode(java.lang.String path,
java.lang.String scriptingLang,
java.lang.String namespace)
Universal Scripting Language Support, see PIG-928
|
void |
registerFunction(java.lang.String function,
FuncSpec funcSpec)
Defines an alias for the given function spec.
|
void |
registerJar(java.lang.String name)
Registers a jar file.
|
void |
registerQuery(java.lang.String query)
Register a query with the Pig runtime.
|
void |
registerQuery(java.lang.String query,
int startLine)
Register a query with the Pig runtime.
|
void |
registerScript(java.io.InputStream in)
Register a pig script from InputStream source which is more general and extensible
the pig script can be from local file, then you can use FileInputStream.
|
void |
registerScript(java.io.InputStream in,
java.util.List<java.lang.String> paramsFiles)
Register a pig script from InputStream source which is more general and extensible
the pig script can be from local file, then you can use FileInputStream.
|
void |
registerScript(java.io.InputStream in,
java.util.Map<java.lang.String,java.lang.String> params)
Register a pig script from InputStream source which is more general and extensible
the pig script can be from local file, then you can use FileInputStream.
|
void |
registerScript(java.io.InputStream in,
java.util.Map<java.lang.String,java.lang.String> params,
java.util.List<java.lang.String> paramsFiles)
Register a pig script from InputStream.
The pig script can be from local file, then you can use FileInputStream. |
void |
registerScript(java.lang.String fileName)
Register a query with the Pig runtime.
|
void |
registerScript(java.lang.String fileName,
java.util.List<java.lang.String> paramsFiles)
Register a pig script file.
|
void |
registerScript(java.lang.String fileName,
java.util.Map<java.lang.String,java.lang.String> params)
Register a pig script file.
|
void |
registerScript(java.lang.String fileName,
java.util.Map<java.lang.String,java.lang.String> params,
java.util.List<java.lang.String> paramsFiles)
Register a pig script file.
|
void |
registerStreamingCommand(java.lang.String commandAlias,
StreamingCommand command)
Defines an alias for the given streaming command.
|
boolean |
renameFile(java.lang.String source,
java.lang.String target)
Rename a file.
|
static void |
resetScope() |
void |
setBatchOn()
Starts batch execution mode.
|
void |
setDefaultParallel(int p)
Set the default parallelism for this job
|
void |
setJobName(java.lang.String name)
Set the name of the job.
|
void |
setJobPriority(java.lang.String priority)
Set Hadoop job priority.
|
void |
setSkipParseInRegisterForBatch(boolean skipParseInRegisterForBatch)
Set whether to skip parsing while registering the query in batch mode
|
void |
setValidateEachStatement(boolean validateEachStatement)
This can be called to indicate if the query is being parsed/compiled
in a mode that expects each statement to be validated as it is
entered, instead of just doing it once for whole script.
|
void |
shutdown()
Reclaims resources used by this instance of PigServer.
|
ExecJob |
store(java.lang.String id,
java.lang.String filename)
Executes a Pig Latin script up to and including indicated alias and stores the resulting
records into a file.
|
ExecJob |
store(java.lang.String id,
java.lang.String filename,
java.lang.String func)
Executes a Pig Latin script up to and including indicated alias and stores the resulting
records into a file.
|
protected final org.apache.commons.logging.Log log
public static final java.lang.String PRETTY_PRINT_SCHEMA_PROPERTY
protected final java.util.Deque<PigServer.Graph> graphs
protected final PigContext pigContext
protected final java.lang.String scope
public PigServer(java.lang.String execTypeString) throws ExecException, java.io.IOException
execTypeString
- can be 'mapreduce' or 'local'. Local mode will
use Hadoop's local job runner to execute the job on the local machine.
Mapreduce mode will connect to a cluster to execute the job. If
execTypeString is not one of these two, Pig will deduce the ExecutionEngine
if it is on the classpath and use it for the backend execution.ExecException
java.io.IOException
public PigServer(java.lang.String execTypeString, java.util.Properties properties) throws ExecException, java.io.IOException
ExecException
java.io.IOException
public PigServer(java.util.Properties properties) throws ExecException, java.io.IOException
ExecException
java.io.IOException
public PigServer(ExecType execType) throws ExecException
execType
- execution type to start the engine. Local mode will
use Hadoop's local job runner to execute the job on the local machine.
Mapreduce mode will connect to a cluster to execute the job.ExecException
public PigServer(ExecType execType, java.util.Properties properties) throws ExecException
ExecException
public PigServer(ExecType execType, org.apache.hadoop.conf.Configuration conf) throws ExecException
ExecException
public PigServer(PigContext context) throws ExecException
ExecException
public PigServer(PigContext context, boolean connect) throws ExecException
ExecException
public static void resetScope()
public PigContext getPigContext()
public PigServer.Graph getCurrentDAG()
public void debugOn()
public void debugOff()
public void setDefaultParallel(int p)
p
- default number of reducers to use for this job.public void setBatchOn()
public boolean isBatchOn()
public boolean isBatchEmpty() throws FrontendException
FrontendException
public void parseAndBuild() throws java.io.IOException
executeBatch(boolean)
with
argument as false. Do Not use executeBatch()
after
calling this method as that will re-parse and build the script.java.io.IOException
public java.util.List<ExecJob> executeBatch() throws java.io.IOException
java.io.IOException
public java.util.List<ExecJob> executeBatch(boolean parseAndBuild) throws java.io.IOException
parseAndBuild()
before. Pass false as an argument in which case.parseAndBuild
- java.io.IOException
protected java.util.List<ExecJob> getJobs(PigStats stats)
stats
- public void discardBatch() throws FrontendException
FrontendException
public void addPathToSkip(java.lang.String path)
path
- path to be skippedpublic void registerFunction(java.lang.String function, FuncSpec funcSpec)
function
- - the new function alias to define.funcSpec
- - the FuncSpec object representing the name of
the function class and any arguments to constructor.public void registerStreamingCommand(java.lang.String commandAlias, StreamingCommand command)
commandAlias
- - the new command alias to definecommand
- - streaming command to be executedpublic void registerJar(java.lang.String name) throws java.io.IOException
name
- of the jar file to registerjava.io.IOException
public void registerCode(java.lang.String path, java.lang.String scriptingLang, java.lang.String namespace) throws java.io.IOException
path
- path of the script filescriptingLang
- language keyword or scriptingEngine used to interpret the scriptnamespace
- namespace defined for functions of this scriptjava.io.IOException
public void registerQuery(java.lang.String query, int startLine) throws java.io.IOException
query
- a Pig Latin expression to be evaluated.startLine
- line number of the query within the whole scriptjava.io.IOException
public void registerQuery(java.lang.String query) throws java.io.IOException
registerQuery(String, int)
with startLine set to 1.query
- a Pig Latin expression to be evaluated.java.io.IOException
public void registerScript(java.io.InputStream in) throws java.io.IOException
in
- java.io.IOException
public void registerScript(java.io.InputStream in, java.util.Map<java.lang.String,java.lang.String> params) throws java.io.IOException
in
- params
- the key is the parameter name, and the value is the parameter valuejava.io.IOException
public void registerScript(java.io.InputStream in, java.util.List<java.lang.String> paramsFiles) throws java.io.IOException
in
- paramsFiles
- files which have the parameter settingjava.io.IOException
public void registerScript(java.io.InputStream in, java.util.Map<java.lang.String,java.lang.String> params, java.util.List<java.lang.String> paramsFiles) throws java.io.IOException
in
- params
- the key is the parameter name, and the value is the parameter valueparamsFiles
- files which have the parameter settingjava.io.IOException
protected java.util.List<java.lang.String> paramMapToList(java.util.Map<java.lang.String,java.lang.String> params)
protected PigServer.Graph getClonedGraph() throws java.io.IOException
java.io.IOException
public void registerScript(java.lang.String fileName) throws java.io.IOException
fileName
- file to read query from.java.io.IOException
public void registerScript(java.lang.String fileName, java.util.Map<java.lang.String,java.lang.String> params) throws java.io.IOException
fileName
- pig script fileparams
- the key is the parameter name, and the value is the parameter valuejava.io.IOException
public void registerScript(java.lang.String fileName, java.util.List<java.lang.String> paramsFiles) throws java.io.IOException
fileName
- pig script fileparamsFiles
- files which have the parameter settingjava.io.IOException
public void registerScript(java.lang.String fileName, java.util.Map<java.lang.String,java.lang.String> params, java.util.List<java.lang.String> paramsFiles) throws java.io.IOException
fileName
- pig scriptparams
- the key is the parameter name, and the value is the parameter valueparamsFiles
- files which have the parameter settingjava.io.IOException
public void printAliases() throws FrontendException
FrontendException
public Schema dumpSchema(java.lang.String alias) throws java.io.IOException
alias
- Alias whose schema will be written outjava.io.IOException
public Schema dumpSchemaNested(java.lang.String alias, java.lang.String nestedAlias) throws java.io.IOException
alias
- Alias whose schema has nestedAliasnestedAlias
- Alias whose schema will be written outjava.io.IOException
public void setJobName(java.lang.String name)
name
- of jobpublic void setJobPriority(java.lang.String priority)
priority
- valid values are found in JobPriority
public java.util.Iterator<Tuple> openIterator(java.lang.String id) throws java.io.IOException
PigServer server = new PigServer(); server.registerQuery("A = load 'foo';"); server.registerQuery("B = filter A by $0 > 0;"); server.registerQuery("C = order B by $1;");Then
server.openIterator("B");filtered but unsorted data will be returned. If instead a user does
server.openIterator("C");filtered and sorted data will be returned.
id
- Alias to open iterator forjava.io.IOException
public ExecJob store(java.lang.String id, java.lang.String filename) throws java.io.IOException
PigServer server = new PigServer(); server.registerQuery("A = load 'foo';"); server.registerQuery("B = filter A by $0 > 0;"); server.registerQuery("C = order B by $1;");Then
server.store("B", "bar");filtered but unsorted data will be stored to the file bar. If instead a user does
server.store("C", "bar");filtered and sorted data will be stored to the file bar. Equivalent to calling
store(String, String, String)
with
org.apache.pig.PigStorage as the store function.id
- The alias to storefilename
- The file to which to store toExecJob
containing information about this jobjava.io.IOException
public ExecJob store(java.lang.String id, java.lang.String filename, java.lang.String func) throws java.io.IOException
PigServer server = new PigServer(); server.registerQuery("A = load 'foo';"); server.registerQuery("B = filter A by $0 > 0;"); server.registerQuery("C = order B by $1;");Then
server.store("B", "bar", "mystorefunc");filtered but unsorted data will be stored to the file bar using mystorefunc. If instead a user does
server.store("C", "bar", "mystorefunc");filtered and sorted data will be stored to the file bar using mystorefunc.
id
- The alias to storefilename
- The file to which to store tofunc
- store function to useExecJob
containing information about this jobjava.io.IOException
public void explain(java.lang.String alias, java.io.PrintStream stream) throws java.io.IOException
alias
- Name of alias to explain.stream
- PrintStream to write explanation to.java.io.IOException
- if the requested alias cannot be found.public void explain(java.lang.String alias, java.lang.String format, boolean verbose, boolean markAsExecute, java.io.PrintStream lps, java.io.PrintStream eps, java.io.File dir, java.lang.String suffix) throws java.io.IOException
alias
- Name of alias to explain.format
- Format in which the explain should be printed. If text, then the plan will
be printed in plain text. Otherwise, the execution plan will be printed in
DOT format.verbose
- Controls the amount of information printedmarkAsExecute
- When set will treat the explain like a
call to execute in the respoect that all the pending stores are
marked as complete.lps
- Stream to print the logical treeeps
- Stream to print the ExecutionEngine trees. If null, then will print to filesdir
- Directory to print ExecutionEngine trees. If null, will use epssuffix
- Suffix of file namesjava.io.IOException
- if the requested alias cannot be found.public long capacity() throws java.io.IOException
java.io.IOException
public long fileSize(java.lang.String filename) throws java.io.IOException
filename
- java.io.IOException
public boolean existsFile(java.lang.String filename) throws java.io.IOException
filename
- to testjava.io.IOException
public boolean deleteFile(java.lang.String filename) throws java.io.IOException
filename
- to deletejava.io.IOException
public boolean renameFile(java.lang.String source, java.lang.String target) throws java.io.IOException
source
- file to renametarget
- new file namejava.io.IOException
public boolean mkdirs(java.lang.String dirs) throws java.io.IOException
dirs
- directory to makejava.io.IOException
public java.lang.String[] listPaths(java.lang.String dir) throws java.io.IOException
dir
- name of directory to listjava.io.IOException
public java.util.Map<java.lang.String,LogicalPlan> getAliases()
public void shutdown()
public java.util.Set<java.lang.String> getAliasKeySet()
public java.util.Map<Operator,DataBag> getExamples(java.lang.String alias) throws java.io.IOException
java.io.IOException
public void printHistory(boolean withNumbers)
protected PigStats launchPlan(LogicalPlan lp, java.lang.String jobName) throws ExecException, FrontendException
lp
- The logical planjobName
- A String containing the job name to be usedExecException
FrontendException
public LogicalPlanData getLogicalPlanData()
registerQuery(String)
or registerScript(InputStream)
methods.public void setValidateEachStatement(boolean validateEachStatement)
validateEachStatement
- public void setSkipParseInRegisterForBatch(boolean skipParseInRegisterForBatch)
skipParseInRegisterForBatch
- public java.lang.String getLastRel()
public boolean isDebugOn()
public java.lang.String getJobName()
public java.lang.String getJobPriority()
Copyright © 2007-2017 The Apache Software Foundation