• 如果您觉得本站非常有看点,那么赶紧使用Ctrl+D 收藏吧

Java VisitorException类的典型用法和代码示例

java 1次浏览

本文整理汇总了Java中org.apache.pig.impl.plan.VisitorException的典型用法代码示例。如果您正苦于以下问题:Java VisitorException类的具体用法?Java VisitorException怎么用?Java VisitorException使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

VisitorException类属于org.apache.pig.impl.plan包,在下文中一共展示了VisitorException类的40个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。

示例1: visitFRJoin

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitFRJoin(POFRJoin join) throws VisitorException {
	List<FileSpec> newrepl = new ArrayList<FileSpec>();
	
	// Extract the files.
	for (FileSpec f : join.getReplFiles()) {
		if (f == null) {
			newrepl.add(f);
			continue;
		}
		
		// Use the rFileMap to swap things.
		newrepl.add(rFileMap.get(f));
	}
	
	join.setReplFiles(newrepl.toArray(join.getReplFiles()));
}
 

开发者ID:JamesLampton,
项目名称:piggybank-squeal,
代码行数:18,
代码来源:ReplJoinFixer.java

示例2: visitTezPlanContainerNode

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitTezPlanContainerNode(TezPlanContainerNode tezPlanNode) throws VisitorException {
    TezScriptState ss = TezScriptState.get();
    TezDAGScriptInfo dagScriptInfo = ss.setDAGScriptInfo(tezPlanNode);
    TezDAGStats.JobGraphBuilder jobGraphBuilder = new TezDAGStats.JobGraphBuilder(tezPlanNode.getTezOperPlan(), dagScriptInfo);
    jobGraphBuilder.visit();
    TezDAGStats currStats = new TezDAGStats(tezPlanNode.getOperatorKey().toString(), jobGraphBuilder.getJobPlan(), jobGraphBuilder.getTezVertexStatsMap());
    currStats.setAlias(dagScriptInfo);
    jobPlan.add(currStats);
    List<TezPlanContainerNode> preds = getPlan().getPredecessors(tezPlanNode);
    if (preds != null) {
        for (TezPlanContainerNode pred : preds) {
            TezDAGStats predStats = tezDAGStatsMap.get(pred.getOperatorKey().toString());
            if (!jobPlan.isConnected(predStats, currStats)) {
                jobPlan.connect(predStats, currStats);
            }
        }
    }
    tezDAGStatsMap.put(tezPlanNode.getOperatorKey().toString(), currStats);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:21,
代码来源:TezPigScriptStats.java

示例3: getByteArrayPositions

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
/**
 * Gets the positions in the schema which are byte arrays
 * @param func
 *
 * @param s -
 *            input schema
 * @throws VisitorException
 */
private List<Integer> getByteArrayPositions(UserFuncExpression func, Schema s)
        throws VisitorException {
    List<Integer> result = new ArrayList<Integer>();
    for (int i = 0; i < s.size(); i++) {
        try {
            FieldSchema fs = s.getField(i);
            if (fs.type == DataType.BYTEARRAY) {
                result.add(i);
            }
        } catch (FrontendException fee) {
            int errCode = 1043;
            String msg = "Unable to retrieve field schema.";
            throw new TypeCheckerException(func, msg, errCode, PigException.INPUT, fee);            }
    }
    return result;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:25,
代码来源:TypeCheckingExpVisitor.java

示例4: addUidLoadFuncToMap

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
/**
 * Add given uid, loadFuncSpec to mapping
 * @param uid
 * @param loadFuncSpec
 * @throws VisitorException 
 */
private void addUidLoadFuncToMap(long uid, FuncSpec loadFuncSpec)
throws VisitorException{
    if(loadFuncSpec == null){
        return;
    }
    //ensure that uid always matches to same load func
    FuncSpec curFuncSpec = uid2LoadFuncMap.get(uid);
    if(curFuncSpec == null){
        uid2LoadFuncMap.put(uid, loadFuncSpec);
    }else if(! curFuncSpec.equals(loadFuncSpec)){
        String msg = "Bug: uid mapped to two different load functions : " +
        curFuncSpec + " and " + loadFuncSpec;
        throw new VisitorException(msg,2262, PigException.BUG) ;
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:22,
代码来源:LineageFindRelVisitor.java

示例5: handlePackage

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private void handlePackage(MapReduceOper mr, POPackage pkg) throws VisitorException {
    // the LocalRearrange(s) could either be in the map of this MapReduceOper
    // OR in the reduce of predecessor MapReduceOpers
    int lrFound = 0;
    
    lrFound = patchPackage(mr.mapPlan, pkg);
    if(lrFound != pkg.getNumInps()) {
        // we did not find the LocalRearrange(s) in the map plan
        // let's look in the predecessors
        List<MapReduceOper> preds = this.mPlan.getPredecessors(mr);
        for (Iterator<MapReduceOper> it = preds.iterator(); it.hasNext();) {
            MapReduceOper mrOper = it.next();
            if (mrOper.isLimitOnly() && !mPlan.getPredecessors(mrOper).get(0).isGlobalSort())
                mrOper = this.mPlan.getPredecessors(mrOper).get(0);
            lrFound += patchPackage(mrOper.reducePlan, pkg);
            if(lrFound == pkg.getNumInps()) {
                break;
            }     
        }
    }
    if(lrFound != pkg.getNumInps()) {
        int errCode = 2086;
        String msg = "Unexpected problem during optimization. Could not find all LocalRearrange operators.";
        throw new OptimizerException(msg, errCode, PigException.BUG);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:27,
代码来源:POPackageAnnotator.java

示例6: visitNative

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitNative(PONative op) throws VisitorException{
    // We will explode the native operator here to add a new MROper for native Mapreduce job
    try{
        // add a map reduce boundary
        MapReduceOper nativeMROper = getNativeMROp(op.getNativeMRjar(), op.getParams());
        MRPlan.add(nativeMROper);
        MRPlan.connect(curMROp, nativeMROper);
        phyToMROpMap.put(op, nativeMROper);
        curMROp = nativeMROper;
    }catch(Exception e){
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:17,
代码来源:MRCompiler.java

示例7: start

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
void start(PigContext pigContext, JobClient jobClient, 
        JobControlCompiler jcc, MROperPlan mrPlan) {
    
    if (pigContext == null || jobClient == null || jcc == null) {
        LOG.warn("invalid params: " + pigContext + jobClient + jcc);
        return;
    }
    
    this.pigContext = pigContext;
    this.jobClient = jobClient;
    this.jcc = jcc;         
    
    // build job DAG with job ids assigned to null 
    try {
        new JobGraphBuilder(mrPlan).visit();
    } catch (VisitorException e) {
        LOG.warn("unable to build job plan", e);
    }
    
    startTime = System.currentTimeMillis();
    userId = System.getProperty("user.name");
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:23,
代码来源:SimplePigStats.java

示例8: handlePackage

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private void handlePackage(TezOperator pkgTezOp, POPackage pkg) throws VisitorException {
    // the LocalRearrange(s) must be in the plan of a predecessor tez op
    int lrFound = 0;
    List<TezOperator> preds = this.mPlan.getPredecessors(pkgTezOp);
    for (Iterator<TezOperator> it = preds.iterator(); it.hasNext();) {
        TezOperator predTezOp = it.next();
        if (predTezOp.isVertexGroup()) {
            // Just get one of the inputs to vertex group
            predTezOp = getPlan().getOperator(predTezOp.getVertexGroupMembers().get(0));
        }
        lrFound += patchPackage(predTezOp, pkgTezOp, pkg);
        if(lrFound == pkg.getNumInps()) {
            break;
        }
    }

    if(lrFound != pkg.getNumInps()) {
        int errCode = 2086;
        String msg = "Unexpected problem during optimization. Could not find all LocalRearrange operators.";
        throw new OptimizerException(msg, errCode, PigException.BUG);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:23,
代码来源:TezPOPackageAnnotator.java

示例9: handlePackage

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private void handlePackage(MapReduceOper mr, POPackage pkg) throws VisitorException {
    // the LocalRearrange(s) could either be in the map of this MapReduceOper
    // OR in the reduce of predecessor MapReduceOpers
    int lrFound = 0;

    lrFound = patchPackage(mr.mapPlan, pkg);
    if(lrFound != pkg.getNumInps()) {
        // we did not find the LocalRearrange(s) in the map plan
        // let's look in the predecessors
        List<MapReduceOper> preds = this.mPlan.getPredecessors(mr);
        for (Iterator<MapReduceOper> it = preds.iterator(); it.hasNext();) {
            MapReduceOper mrOper = it.next();
            if (mrOper.isLimitOnly() && !mPlan.getPredecessors(mrOper).get(0).isGlobalSort())
                mrOper = this.mPlan.getPredecessors(mrOper).get(0);
            lrFound += patchPackage(mrOper.reducePlan, pkg);
            if(lrFound == pkg.getNumInps()) {
                break;
            }
        }
    }
    if(lrFound != pkg.getNumInps()) {
        int errCode = 2086;
        String msg = "Unexpected problem during optimization. Could not find all LocalRearrange operators.";
        throw new OptimizerException(msg, errCode, PigException.BUG);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:27,
代码来源:POPackageAnnotator.java

示例10: visitConstant

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitConstant(ConstantExpression cnst) throws VisitorException {
    if (cnst.getRequestedParallelism() == -1) {
        Object obj = cnst.getValue();
        if (obj instanceof Integer) {
            if (replaced) {
                // sample job should have only one ConstantExpression
                throw new VisitorException("Invalid reduce plan: more " +
                               "than one ConstantExpression found in sampling job");
            }
            cnst.setValue(rp);
            cnst.setRequestedParallelism(rp);
            replaced = true;
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:17,
代码来源:JobControlCompiler.java

示例11: visitRank

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
/**
 * In case of PORank, it is closed any other previous job (containing
 * POCounter as a leaf) and PORank is added on map phase.
 **/
@Override
public void visitRank(PORank op) throws VisitorException {
    try{
        FileSpec fSpec = getTempFileSpec();
        MapReduceOper prevMROper = endSingleInputPlanWithStr(fSpec);

        curMROp = startNew(fSpec, prevMROper);
        curMROp.mapPlan.addAsLeaf(op);

        phyToMROpMap.put(op, curMROp);
    }catch(Exception e){
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:21,
代码来源:MRCompiler.java

示例12: byteArrayFound

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
/**
 * Checks to see if any field of the input schema is a byte array
 * @param func
 * @param s - input schema
 * @return true if found else false
 * @throws VisitorException
 */
private boolean byteArrayFound(UserFuncExpression func, Schema s) throws VisitorException {
    for(int i=0;i<s.size();i++){
        try {
            FieldSchema fs=s.getField(i);
            if(fs == null)
                return false;
            if(fs.type==DataType.BYTEARRAY){
                return true;
            }
        } catch (FrontendException fee) {
            int errCode = 1043;
            String msg = "Unable to retrieve field schema.";
            throw new TypeCheckerException(func, msg, errCode, PigException.INPUT, fee);
        }
    }
    return false;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:25,
代码来源:TypeCheckingExpVisitor.java

示例13: reInitialize

点赞 3

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private void reInitialize() {
    if(planLeafOps != null) {
        noItems = planLeafOps.length;
        resultTypes = new byte[noItems];
        for (int i = 0; i < resultTypes.length; i++) {
            resultTypes[i] = planLeafOps[i].getResultType();
        }
    } else {
        noItems = 0;
        resultTypes = null;
    }

    if(inputPlans != null) {
        for (PhysicalPlan pp : inputPlans) {
            try {
                ResetFinder lf = new ResetFinder(pp, opsToBeReset);
                lf.visit();
            } catch (VisitorException ve) {
                String errMsg = "Internal Error:  Unexpected error looking for nested operators which need to be reset in FOREACH";
                throw new RuntimeException(errMsg, ve);
            }
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:25,
代码来源:POForEach.java

示例14: mergeMapReduceSplittees

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private int mergeMapReduceSplittees(List<MapReduceOper> mapReducers,
        MapReduceOper splitter, POSplit splitOp) throws VisitorException {

    List<MapReduceOper> mergeList = getMergeList(splitter, mapReducers);

    if (mergeList.size() <= 1) {

        // chose one to merge, prefer the one with a combiner
        MapReduceOper mapReducer = mapReducers.get(0);
        for (MapReduceOper mro : mapReducers) {
            if (!mro.combinePlan.isEmpty()) {
                mapReducer = mro;
                break;
            }
        }
        mergeList.clear();
        mergeList.add(mapReducer);
    }

    if (mergeList.size() == 1) {
        mergeSingleMapReduceSplittee(mergeList.get(0), splitter, splitOp);
    } else {
        mergeAllMapReduceSplittees(mergeList, splitter, splitOp);
    }

    return mergeList.size();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:28,
代码来源:MultiQueryOptimizer.java

示例15: getOrderbySamplingAggregationJob

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private Pair<TezOperator,Integer> getOrderbySamplingAggregationJob(
        POSort inpSort,
        int rp) throws PlanException, VisitorException, ExecException {

    POSort sort = new POSort(inpSort.getOperatorKey(), inpSort
            .getRequestedParallelism(), null, inpSort.getSortPlans(),
            inpSort.getMAscCols(), inpSort.getMSortFunc());
    sort.addOriginalLocation(inpSort.getAlias(), inpSort.getOriginalLocations());

    // Turn the asc/desc array into an array of strings so that we can pass it
    // to the FindQuantiles function.
    List<Boolean> ascCols = inpSort.getMAscCols();
    String[] ascs = new String[ascCols.size()];
    for (int i = 0; i < ascCols.size(); i++) ascs[i] = ascCols.get(i).toString();
    // check if user defined comparator is used in the sort, if so
    // prepend the name of the comparator as the first fields in the
    // constructor args array to the FindQuantiles udf
    String[] ctorArgs = ascs;
    if(sort.isUDFComparatorUsed) {
        String userComparatorFuncSpec = sort.getMSortFunc().getFuncSpec().toString();
        ctorArgs = new String[ascs.length + 1];
        ctorArgs[0] = USER_COMPARATOR_MARKER + userComparatorFuncSpec;
        for(int j = 0; j < ascs.length; j++) {
            ctorArgs[j+1] = ascs[j];
        }
    }

    return getSamplingAggregationJob(sort, rp, null, FindQuantilesTez.class.getName(), ctorArgs);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:30,
代码来源:TezCompiler.java

示例16: convert

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
public void convert() {
		// Start walking.
		try {
			// Pull out any static subtrees from the execution plan.
			StaticPlanFixer spf = new StaticPlanFixer(plan, pc);
			spf.convert();
			// Pull out the replicated join creation plan.
			ReplJoinFixer rjf = new ReplJoinFixer(plan, spf.getStaticPlan(), pc);
			rjf.convert();
			if (rjf.getReplFileMap().size() > 0) {
				splan.setReplFileMap(rjf.getReplFileMap());
			}
			if (spf.getStaticPlan().size() > 0) {
				splan.setStaticPlan(spf.getStaticPlan());
			}
			
			visit();
			splan.setRootMap(rootMap);
			
//			System.out.println("ReplFiles: " + splan.replFiles);
			if (missingRoots.size() > 0) {
				// We have some paths that aren't attached to the plan.
				log.warn("Missing roots: " + missingRoots);
			}
		} catch (VisitorException e) {
			e.printStackTrace();
		}
//		System.out.println("here");
	}
 

开发者ID:JamesLampton,
项目名称:piggybank-squeal,
代码行数:30,
代码来源:MRtoSConverter.java

示例17: printToString

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
/***
 * This method has to be called after the visit is totally finished only
 * @throws IOException 
 */
public String printToString() throws IOException {
    try {
        visit() ;
    } catch(VisitorException vse) {
        throw new AssertionError("Error while transforming type graph to text") ;
    }
    return sb.toString() ;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:13,
代码来源:OptimizeLimitPlanPrinter.java

示例18: processLoadAndParallelism

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
public static void processLoadAndParallelism(TezOperPlan tezPlan, PigContext pc) throws VisitorException {
    if (!pc.inExplain && !pc.inDumpSchema) {
        LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
        loaderStorer.visit();

        ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
        parallelismSetter.visit();
        tezPlan.setEstimatedParallelism(parallelismSetter.getEstimatedTotalParallelism());
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:11,
代码来源:TezLauncher.java

示例19: toString

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public String toString() {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(baos);
    SPrinter printer = new SPrinter(ps, this, null);
    printer.setVerbose(true);
    try {
        printer.visit();
    } catch (VisitorException e) {
        throw new RuntimeException("Unable to get String representation of plan:" + e );
    }
    return baos.toString();
}
 

开发者ID:JamesLampton,
项目名称:piggybank-squeal,
代码行数:14,
代码来源:SOperPlan.java

示例20: mergeSingleMapReduceSplittee

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private void mergeSingleMapReduceSplittee(MapReduceOper mapReduce, 
        MapReduceOper splitter, POSplit splitOp) throws VisitorException {
    
    PhysicalPlan splitterPl = splitter.mapPlan;
    PhysicalOperator leaf = splitterPl.getLeaves().get(0);
    PhysicalOperator storeOp = splitterPl.getLeaves().get(0);
    List<PhysicalOperator> storePreds = splitterPl.getPredecessors(storeOp);  
                    
    PhysicalPlan pl = mapReduce.mapPlan;
    PhysicalOperator load = pl.getRoots().get(0);
    pl.remove(load);
    
    splitOp.addPlan(pl);
                          
    splitter.setMapDone(true);
    splitter.reducePlan = mapReduce.reducePlan;
    splitter.setReduceDone(true);
    splitter.combinePlan = mapReduce.combinePlan;
    splitter.customPartitioner = mapReduce.customPartitioner;
            
    // replace store operator in the splitter with split operator
    if (leaf instanceof POStore) {                            
        splitOp.setInputs(storePreds);
        try {
            splitterPl.replace(storeOp, splitOp);;
        } catch (PlanException e) {
            int errCode = 2132;
            String msg = "Internal Error. Unable to replace store with split operator for optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG, e);
        }  
    }
    
    removeAndReconnect(mapReduce, splitter);           
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:35,
代码来源:MultiQueryOptimizer.java

示例21: visitPOForEach

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitPOForEach(POForEach fe) throws VisitorException {
    // we need to allow foreach as input for distinct
    // but don't want it for other things (why?). So lets
    // flag the presence of Foreach and if this is present
    // with a distinct agg, it will be allowed.
    sawForeach = true;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:9,
代码来源:CombinerOptimizer.java

示例22: toString

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public String toString() {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(baos);
    MRPrinter printer = new MRPrinter(ps, this);
    printer.setVerbose(true);
    try {
        printer.visit();
    } catch (VisitorException e) {
        // TODO Auto-generated catch block
        throw new RuntimeException("Unable to get String representation of plan:" + e );
    }
    return baos.toString();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:15,
代码来源:MROperPlan.java

示例23: visitLocalRearrange

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
public void visitLocalRearrange(POLocalRearrange lr) throws VisitorException{
    List<PhysicalPlan> inpPlans = lr.getPlans();
    for (PhysicalPlan plan : inpPlans) {
        pushWalker(mCurrentWalker.spawnChildWalker(plan));
        visit();
        popWalker();
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:9,
代码来源:PhyPlanVisitor.java

示例24: visit

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visit() throws VisitorException {
    super.visit();
    // if we saw foreach and distinct agg its ok
    // else if we only saw foreach, mark it as non algebraic
    if (sawForeach && !sawDistinctAgg) {
        sawNonAlgebraic = true;
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:10,
代码来源:CombinerOptimizerUtil.java

示例25: isIntermediateReducer

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
/**
 * Returns true if there are no loads or stores in a TezOperator.
 * To be called only after LoaderProcessor is called
 */
static public boolean isIntermediateReducer(TezOperator tezOper) throws VisitorException {
    boolean intermediateReducer = false;
    LinkedList<POStore> stores = PlanHelper.getPhysicalOperators(tezOper.plan, POStore.class);
    // Not map and not final reducer
    if (stores.size() <= 0 &&
            (tezOper.getLoaderInfo().getLoads() == null || tezOper.getLoaderInfo().getLoads().size() <= 0)) {
        intermediateReducer = true;
    }
    return intermediateReducer;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:15,
代码来源:TezCompilerUtil.java

示例26: assertPlanContains

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private void assertPlanContains(TezOperPlan plan, String[] expectedFiles, int size) throws VisitorException {
    TezPOUserFuncVisitor udfVisitor = new TezPOUserFuncVisitor(plan);
    udfVisitor.visit();

    List<String> shipFiles = new ArrayList<String>();
    shipFiles.addAll(udfVisitor.getShipFiles());

    Assert.assertEquals(shipFiles.size(), size);
    assertContains(shipFiles, expectedFiles);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:11,
代码来源:TestLoaderStorerShipCacheFilesTez.java

示例27: depthFirstPP

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@SuppressWarnings("unchecked")
protected String depthFirstPP() throws VisitorException {
    StringBuilder sb = new StringBuilder();
    List<O> leaves = mPlan.getLeaves();
    Collections.sort(leaves);
    for (O leaf : leaves) {
        sb.append(depthFirst(leaf));
        sb.append("\n");
    }
    sb.delete(sb.length() - "\n".length(), sb.length());
    sb.delete(sb.length() - "\n".length(), sb.length());
    return sb.toString();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:14,
代码来源:PlanPrinter.java

示例28: visitCollectedGroup

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
public void visitCollectedGroup(POCollectedGroup mg) throws VisitorException{
    List<PhysicalPlan> inpPlans = mg.getPlans();
    for (PhysicalPlan plan : inpPlans) {
        pushWalker(mCurrentWalker.spawnChildWalker(plan));
        visit();
        popWalker();
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:9,
代码来源:PhyPlanVisitor.java

示例29: visitPreCombinerLocalRearrange

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
/**
 * @param preCombinerLocalRearrange
 */
public void visitPreCombinerLocalRearrange(
        POPreCombinerLocalRearrange preCombinerLocalRearrange) throws VisitorException {
    List<PhysicalPlan> inpPlans = preCombinerLocalRearrange.getPlans();
    for (PhysicalPlan plan : inpPlans) {
        pushWalker(mCurrentWalker.spawnChildWalker(plan));
        visit();
        popWalker();
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:13,
代码来源:PhyPlanVisitor.java

示例30: visitStore

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitStore(POStore sto) throws VisitorException {
    FileSpec spec = sto.getSFile();
    if (oldSpec.equals(spec)) {
        sto.setSFile(newSpec);
    }           
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:8,
代码来源:MRCompiler.java

示例31: mergeMapReduceSplittees

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private int mergeMapReduceSplittees(List<MapReduceOper> mapReducers, 
        MapReduceOper splitter, POSplit splitOp) throws VisitorException {
            
    List<MapReduceOper> mergeList = getMergeList(splitter, mapReducers);

    if (mergeList.size() <= 1) {

        // chose one to merge, prefer the one with a combiner
        MapReduceOper mapReducer = mapReducers.get(0);
        for (MapReduceOper mro : mapReducers) {
            if (!mro.combinePlan.isEmpty()) {
                mapReducer = mro;
                break;
            }
        }
        mergeList.clear();
        mergeList.add(mapReducer);
    } 
                     
    if (mergeList.size() == 1) {
        mergeSingleMapReduceSplittee(mergeList.get(0), splitter, splitOp);
    } else {                                   
        mergeAllMapReduceSplittees(mergeList, splitter, splitOp);
    }
    
    return mergeList.size();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:28,
代码来源:MultiQueryOptimizer.java

示例32: visitProject

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitProject(POProject proj) throws VisitorException {
    if (proj.getResultType() == DataType.BAG) {
        // IMPORTANT ASSUMPTION:
        // we should be calling this visitor only for
        // fixing up the projects in the map's foreach
        // inner plan. In the map side, we are dealing
        // with single tuple bags - so set the flag in
        // the project to use single tuple bags. If in
        // future we don't have single tuple bags in the
        // input to map's foreach, we should NOT be doing
        // this!
        proj.setResultSingleTupleBag(true);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:16,
代码来源:CombinerOptimizerUtil.java

示例33: getSplitBranch

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private static PhysicalPlan getSplitBranch(POSplit split, String outputKey) throws VisitorException {
    List<PhysicalPlan> plans = split.getPlans();
    for (PhysicalPlan plan : plans) {
        LinkedList<POLocalRearrangeTez> lrs = PlanHelper.getPhysicalOperators(plan, POLocalRearrangeTez.class);
        if (!lrs.isEmpty()) {
            return plan;
        }
        LinkedList<POValueOutputTez> vos = PlanHelper.getPhysicalOperators(plan, POValueOutputTez.class);
        if (!vos.isEmpty()) {
            return plan;
        }
    }
    return null;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:15,
代码来源:TezOperDependencyParallelismEstimator.java

示例34: visit

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visit() throws VisitorException {
    try {
        stream.write(depthFirstPP().getBytes());
    } catch (IOException ioe) {
        int errCode = 2079;
        String msg = "Unexpected error while printing physical plan.";
        throw new VisitorException(msg, errCode, PigException.BUG, ioe);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:11,
代码来源:PlanPrinter.java

示例35: patchPackage

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private int patchPackage(PhysicalPlan plan, POPackage pkg) throws VisitorException {
    LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(plan, pkg);
    lrDiscoverer.visit();
    // let our caller know if we managed to patch
    // the package
    return lrDiscoverer.getLoRearrangeFound();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:8,
代码来源:POPackageAnnotator.java

示例36: visitLoad

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitLoad(POLoad op) throws VisitorException{
    try{
        nonBlocking(op);
        phyToMROpMap.put(op, curMROp);
    }catch(Exception e){
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:12,
代码来源:MRCompiler.java

示例37: visitJoinPackage

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitJoinPackage(POJoinPackage joinPackage) throws VisitorException{
    if (revisit &&  joinPackage.getIllustrator() != null)
        return;
    setIllustrator(joinPackage);
    joinPackage.getForEach().setIllustrator(joinPackage.getIllustrator());
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:8,
代码来源:IllustratorAttacher.java

示例38: visitStream

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
@Override
public void visitStream(POStream op) throws VisitorException{
    try{
        nonBlocking(op);
        phyToMROpMap.put(op, curMROp);
    }catch(Exception e){
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:12,
代码来源:MRCompiler.java

示例39: setIndexOnLRInSplit

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private int setIndexOnLRInSplit(int initial, POSplit splitOp, boolean sameKeyType)
        throws VisitorException {
    int index = initial;
    
    List<PhysicalPlan> pls = splitOp.getPlans();
    for (PhysicalPlan pl : pls) {
        PhysicalOperator leaf = pl.getLeaves().get(0);
        if (leaf instanceof POLocalRearrange) {
            POLocalRearrange lr = (POLocalRearrange)leaf;
            try {
                lr.setMultiQueryIndex(index++); 
            } catch (ExecException e) {                    
                int errCode = 2136;
                String msg = "Internal Error. Unable to set multi-query index for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);                   
            }
            
            // change the map key type to tuple when 
            // multiple splittees have different map key types
            if (!sameKeyType) {
                lr.setKeyType(DataType.TUPLE);
            }
        } else if (leaf instanceof POSplit) {
            POSplit spl = (POSplit)leaf;
            index = setIndexOnLRInSplit(index, spl, sameKeyType);
        }
    }

    return index;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:31,
代码来源:MultiQueryOptimizer.java

示例40: fixScalar

点赞 2

import org.apache.pig.impl.plan.VisitorException; //导入依赖的package包/类
private void fixScalar() throws VisitorException, PlanException {
    // Mapping POStore to POValueOuptut
    Map<POStore, POValueOutputTez> storeSeen = new HashMap<POStore, POValueOutputTez>();

    for (TezOperator tezOp : tezPlan) {
        List<POUserFunc> userFuncs = PlanHelper.getPhysicalOperators(tezOp.plan, POUserFunc.class);
        for (POUserFunc userFunc : userFuncs) {
            if (userFunc.getReferencedOperator()!=null) {  // Scalar
                POStore store = (POStore)userFunc.getReferencedOperator();

                TezOperator from = phyToTezOpMap.get(store);

                FuncSpec newSpec = new FuncSpec(ReadScalarsTez.class.getName(), from.getOperatorKey().toString());
                userFunc.setFuncSpec(newSpec);

                if (storeSeen.containsKey(store)) {
                    storeSeen.get(store).addOutputKey(tezOp.getOperatorKey().toString());
                } else {
                    POValueOutputTez output = new POValueOutputTez(OperatorKey.genOpKey(scope));
                    output.addOutputKey(tezOp.getOperatorKey().toString());
                    from.plan.remove(from.plan.getOperator(store.getOperatorKey()));
                    from.plan.addAsLeaf(output);
                    storeSeen.put(store, output);

                    //Remove unused store filename
                    userFunc.getInputs().remove(1);
                }

                TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, from, tezOp);
                //TODO shared edge once support is available in Tez
                TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.BROADCAST);
            }
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:36,
代码来源:TezCompiler.java


版权声明:本文转自网络文章,转载此文章仅为分享知识,如有侵权,请联系管理员进行删除。
喜欢 (0)