• 如果您觉得本站非常有看点,那么赶紧使用Ctrl+D 收藏吧

Java PhysicalOperator类的典型用法和代码示例

java 1次浏览

本文整理汇总了Java中org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator的典型用法代码示例。如果您正苦于以下问题:Java PhysicalOperator类的具体用法?Java PhysicalOperator怎么用?Java PhysicalOperator使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

PhysicalOperator类属于org.apache.pig.backend.hadoop.executionengine.physicalLayer包,在下文中一共展示了PhysicalOperator类的36个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。

示例1: visitSOp

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@Override
public void visitSOp(StormOper sop) throws VisitorException {
    mStream.println("Storm node " + sop.getOperatorKey().toString() + " type: " + sop.getType() + " alias: " + sop.name() + " parallel: " + sop.getParallelismHint());
    if (sop.getType() == StormOper.OpType.BASIC_PERSIST || sop.getType() == StormOper.OpType.COMBINE_PERSIST) {
    	mStream.println("Backing Store: " + sop.getStateFactoryOpts(pc));
    	if (sop.getWindowOptions() != null) {
    		mStream.println("Window options: " + sop.getWindowOptions());
    	}
    }
    if (sop.plan != null) {
      PlanPrinter<PhysicalOperator, PhysicalPlan> printer = new PlanPrinter<PhysicalOperator, PhysicalPlan>(sop.plan, mStream);
      printer.setVerbose(isVerbose);
      printer.visit();
      mStream.println("\n--------");        	
    }
    mStream.println("----------------");
    mStream.println("");
}
 

开发者ID:JamesLampton,
项目名称:piggybank-squeal,
代码行数:19,
代码来源:SPrinter.java

示例2: setupStore

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
void setupStore(List<PhysicalOperator> list, StormOper sop) {
	for (PhysicalOperator op : list) {
		if (op.getClass().isAssignableFrom(POStore.class)) {
			POStore ps = (POStore) op;
			String fn = ps.getSFile().getFileName();
			if (missingRoots.containsKey(fn)) {
				for (StormOper waiting : missingRoots.remove(fn)) {
					try {
						splan.connect(sop, waiting);
					} catch (PlanException e) {
						e.printStackTrace();
					}
				}
			}
			rootMap.put(fn, sop);
		}
	}
}
 

开发者ID:JamesLampton,
项目名称:piggybank-squeal,
代码行数:19,
代码来源:MRtoSConverter.java

示例3: getCounterOperation

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private POCounter getCounterOperation() {
    PhysicalOperator operator;
    Iterator<PhysicalOperator> it =  this.mapPlan.getLeaves().iterator();

    while(it.hasNext()) {
        operator = it.next();
        if(operator instanceof POCounter)
            return (POCounter) operator;
    }

    it =  this.reducePlan.getLeaves().iterator();

    while(it.hasNext()) {
        operator = it.next();
        if(operator instanceof POCounter)
            return (POCounter) operator;
    }

    return null;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:21,
代码来源:MapReduceOper.java

示例4: configureStorer

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private static POStore configureStorer(JobConf jobConf,
        PhysicalOperator physicalOperator) throws IOException {
    ArrayList<POStore> storeLocations = Lists.newArrayList();
    POStore poStore = (POStore) physicalOperator;
    storeLocations.add(poStore);
    StoreFuncInterface sFunc = poStore.getStoreFunc();
    sFunc.setStoreLocation(poStore.getSFile().getFileName(),
            new org.apache.hadoop.mapreduce.Job(jobConf));
    poStore.setInputs(null);
    poStore.setParentPlan(null);

    jobConf.set(JobControlCompiler.PIG_MAP_STORES,
            ObjectSerializer.serialize(Lists.newArrayList()));
    jobConf.set(JobControlCompiler.PIG_REDUCE_STORES,
            ObjectSerializer.serialize(storeLocations));
    return poStore;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:18,
代码来源:StoreConverter.java

示例5: setLoadDataMap

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void setLoadDataMap() {
    // This function sets up the LO-TO-Data map, eq. class, and lineage for the base data used in the coming runner
    // this must be called after logToDataMap has been properly (re)set and before the runner is started
    if (baseData != null) {
        if (poToEqclassesMap == null)
            poToEqclassesMap = new HashMap<PhysicalOperator, Collection<IdentityHashSet<Tuple>>>();
        else
            poToEqclassesMap.clear();
        for (LOLoad lo : baseData.keySet()) {
            logToDataMap.get(lo).addAll(baseData.get(lo));
            LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
            IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
            equivalenceClasses.add(equivalenceClass);
            for (Tuple t : baseData.get(lo)) {
                lineage.insert(t);
                equivalenceClass.add(t);
            }
            poToEqclassesMap.put(logToPhyMap.get(lo), equivalenceClasses);
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:22,
代码来源:ExampleGenerator.java

示例6: MRCompiler

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
public MRCompiler(PhysicalPlan plan,
        PigContext pigContext) throws MRCompilerException {
    super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
    this.plan = plan;
    this.pigContext = pigContext;
    splitsSeen = new HashMap<OperatorKey, MapReduceOper>();
    MRPlan = new MROperPlan();
    nig = NodeIdGenerator.getGenerator();
    udfFinder = new UDFFinder();
    List<PhysicalOperator> roots = plan.getRoots();
    if((roots == null) || (roots.size() <= 0)) {
        int errCode = 2053;
        String msg = "Internal error. Did not find roots in the physical plan.";
        throw new MRCompilerException(msg, errCode, PigException.BUG);
    }
    scope = roots.get(0).getOperatorKey().getScope();
    messageCollector = new CompilationMessageCollector() ;
    phyToMROpMap = new HashMap<PhysicalOperator, MapReduceOper>();

    fileConcatenationThreshold = Integer.parseInt(pigContext.getProperties()
            .getProperty(FILE_CONCATENATION_THRESHOLD, "100"));
    optimisticFileConcatenation = pigContext.getProperties().getProperty(
            OPTIMISTIC_FILE_CONCATENATION, "false").equals("true");
    LOG.info("File concatenation threshold: " + fileConcatenationThreshold
            + " optimistic? " + optimisticFileConcatenation);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:27,
代码来源:MRCompiler.java

示例7: userFuncArity

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
public void userFuncArity(DataBag input ) throws ExecException {
	String funcSpec = ARITY.class.getName() + "()";
	PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
	List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
	inputs.add(read);
	POUserFunc userFunc = new POUserFunc(new OperatorKey("", r.nextLong()),
			-1, inputs, new FuncSpec(funcSpec));
	Result res = new Result();
	Integer i = null;
	res = userFunc.getNextInteger();
	while (res.returnStatus != POStatus.STATUS_EOP) {
		// System.out.println(res.result);
		int result = (Integer) res.result;
		assertEquals(2, result);
		res = userFunc.getNextInteger();
	}
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:18,
代码来源:TestPOUserFunc.java

示例8: getLeaves

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
public void getLeaves() {
    if (inputPlans != null) {
        int i=-1;
        if(isToBeFlattenedArray == null) {
            isToBeFlattenedArray = new boolean[inputPlans.size()];
        }
        planLeafOps = new PhysicalOperator[inputPlans.size()];
        for(PhysicalPlan p : inputPlans) {
            ++i;
            PhysicalOperator leaf = p.getLeaves().get(0);
            planLeafOps[i] = leaf;
            if(leaf instanceof POProject &&
                    leaf.getResultType() == DataType.TUPLE &&
                     ((POProject)leaf).isProjectToEnd() ) {
                isToBeFlattenedArray[i] = true;
            }
        }
    }
    // we are calculating plan leaves
    // so lets reinitialize
    reInitialize();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:23,
代码来源:POForEach.java

示例9: setUp2

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void setUp2() throws PlanException, ExecException{
    lr = GenPhyOp.topLocalRearrangeOPWithPlanPlain(0,0,db.iterator().next());
    List<PhysicalPlan> plans = lr.getPlans();
    POLocalRearrange lrT = GenPhyOp.topLocalRearrangeOPWithPlanPlain(0, 1, db.iterator().next());
    List<PhysicalPlan> plansT = lrT.getPlans();
    plans.add(plansT.get(0));
    lr.setPlans(plans);

    POProject proj = GenPhyOp.exprProject();
    proj.setColumn(0);
    proj.setResultType(DataType.TUPLE);
    proj.setOverloaded(true);
    Tuple t = new DefaultTuple();
    t.append(db);
    proj.attachInput(t);
    List<PhysicalOperator> inputs = new ArrayList<PhysicalOperator>();
    inputs.add(proj);
    lr.setInputs(inputs);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:20,
代码来源:TestLocalRearrange.java

示例10: accumulateData

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private void accumulateData() throws ExecException {
    int count = 0;
    int length = inputs.size() - 1;
    inputBags = new DataBag[length];
    its = new Iterator[length];
    for (int i = 0; i < length; ++i) {
        PhysicalOperator op = inputs.get(i);
        DataBag bag = BagFactory.getInstance().newDefaultBag();
        inputBags[count] = bag;
        for (Result res = op.getNextTuple(); res.returnStatus != POStatus.STATUS_EOP; res = op
                .getNextTuple()) {
            if (res.returnStatus == POStatus.STATUS_NULL)
                continue;
            if (res.returnStatus == POStatus.STATUS_ERR)
                throw new ExecException(
                        "Error accumulating data in the local Cross operator");
            if (res.returnStatus == POStatus.STATUS_OK)
                bag.add((Tuple) res.result);
        }
        its[count++] = bag.iterator();
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:24,
代码来源:POCross.java

示例11: LineageTrimmingVisitor

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
public LineageTrimmingVisitor(LogicalPlan plan,
        Map<LOLoad, DataBag> baseData,
        ExampleGenerator eg,
        Map<Operator, PhysicalOperator> LogToPhyMap,
        PhysicalPlan physPlan, PigContext pc) throws IOException, InterruptedException {
    super(plan, new PreOrderDepthFirstWalker(plan));
    // this.baseData.putAll(baseData);
    this.baseData = baseData;
    this.plan = plan;
    this.LogToPhyMap = LogToPhyMap;
    this.pc = pc;
    this.physPlan = physPlan;
    this.eg = eg;
    this.inputToDataMap = new HashMap<FileSpec, DataBag>();
    init();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:17,
代码来源:LineageTrimmingVisitor.java

示例12: runPipeline

点赞 3

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private Result runPipeline(PhysicalOperator leaf) throws ExecException {
   
    Result res = null;
    
    while (true) {
        
        res = leaf.getNextTuple();
        
        if (res.returnStatus == POStatus.STATUS_OK ||
                res.returnStatus == POStatus.STATUS_EOP || 
                res.returnStatus == POStatus.STATUS_ERR) {                
            break;
        } else if (res.returnStatus == POStatus.STATUS_NULL) {
            continue;
        } 
    }   
    
    if (res.returnStatus == POStatus.STATUS_EOP) {
        getNext = true;
    }
    return (res.returnStatus == POStatus.STATUS_OK || res.returnStatus == POStatus.STATUS_ERR) ? res : empty;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:23,
代码来源:PODemux.java

示例13: runPipeline

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
/**
 * @param leaf
 * @param collector 
 * @throws ExecException 
 */
protected void runPipeline(PhysicalOperator leaf, ICollector collector) throws ExecException {
    
    while(true)
    {
        Result redRes = leaf.getNextTuple();
        if(redRes.returnStatus==POStatus.STATUS_OK){
            collector.emit(new FValues(null, (Tuple)redRes.result));
            continue;
        }
        
        if(redRes.returnStatus==POStatus.STATUS_EOP) {
            return;
        }
        
        if(redRes.returnStatus==POStatus.STATUS_NULL) {
            continue;
        }
        
        if(redRes.returnStatus==POStatus.STATUS_ERR){
            // remember that we had an issue so that in 
            // close() we can do the right thing
            errorInReduce = true;
            // if there is an errmessage use it
            String msg;
            if(redRes.result != null) {
                msg = "Received Error while " +
                "processing the reduce plan: " + redRes.result;
            } else {
                msg = "Received Error while " +
                "processing the reduce plan.";
            }
            int errCode = 2090;
            throw new ExecException(msg, errCode, PigException.BUG);
        }
    }
}
 

开发者ID:JamesLampton,
项目名称:piggybank-squeal,
代码行数:42,
代码来源:Reduce.java

示例14: warn

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
protected void warn(String msg, Enum warningEnum, Exception e) {
    pigLogger = PhysicalOperator.getPigLogger();
    if(pigLogger != null) {
        pigLogger.warn(this, msg, warningEnum);
    } else {
        log.warn(msg, e);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:10,
代码来源:DefaultAbstractBag.java

示例15: connect

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@Override
public void connect(PhysicalOperator from, PhysicalOperator to)
        throws PlanException {

    super.connect(from, to);
    to.setInputs(getPredecessors(to));
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:8,
代码来源:PhysicalPlan.java

示例16: isSplitteeMergeable

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private boolean isSplitteeMergeable(MapReduceOper splittee) {
    
    // cannot be global sort or limit after sort, they are 
    // using a different partitioner
    if (splittee.isGlobalSort() || splittee.isLimitAfterSort()) {
        log.info("Cannot merge this splittee: " +
        		"it is global sort or limit after sort");
        return false;
    }
    
    // check the plan leaf: only merge local rearrange or split
    PhysicalOperator leaf = splittee.mapPlan.getLeaves().get(0);
    if (!(leaf instanceof POLocalRearrange) && 
            ! (leaf instanceof POSplit)) {
        log.info("Cannot merge this splittee: " +
        		"its map plan doesn't end with LR or Split operator: " 
                + leaf.getClass().getName());
        return false;
    }
       
    // cannot have distinct combiner, it uses a different combiner
    if (splittee.needsDistinctCombiner()) {
        log.info("Cannot merge this splittee: " +
        		"it has distinct combiner.");
        return false;           
    }
    
    return true;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:30,
代码来源:MultiQueryOptimizer.java

示例17: JoinDistributedCacheVisitor

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
public JoinDistributedCacheVisitor(PhysicalPlan plan,
        PigContext pigContext, Configuration conf) {
    super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(
            plan));
    this.pigContext = pigContext;
    this.conf = conf;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:8,
代码来源:JobControlCompiler.java

示例18: findPOPartialAgg

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private PhysicalOperator findPOPartialAgg(PhysicalPlan mapPlan) {
    Iterator<PhysicalOperator> it = mapPlan.iterator();
    while(it.hasNext()){
        PhysicalOperator op = it.next();
        if(op instanceof POPartialAgg){
            return op;
        }
    }
    return null;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:11,
代码来源:TestPOPartialAggPlan.java

示例19: setAccumulative

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@Override
public void setAccumulative() {
    super.setAccumulative();
    for(PhysicalPlan p : inputPlans) {
        Iterator<PhysicalOperator> iter = p.iterator();
        while(iter.hasNext()) {
            PhysicalOperator po = iter.next();
            if (po instanceof ExpressionOperator || po instanceof PODistinct) {
                po.setAccumulative();
            }
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:14,
代码来源:POForEach.java

示例20: setInputs

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@Override
public void setInputs(List<PhysicalOperator> inputs) {
    super.setInputs(inputs);
    if (inputs != null) {
        done = new BitSet(inputs.size());
    }
    else {
        done = new BitSet(0) ;
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:11,
代码来源:POUnion.java

示例21: attachBinaryComparisonOperator

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void attachBinaryComparisonOperator( BinaryExpression op, 
        BinaryComparisonOperator exprOp ) throws FrontendException {
    // We dont have aliases in ExpressionOperators
    // exprOp.setAlias(op.getAlias());
    
    
    exprOp.setOperandType(op.getLhs().getType());
    exprOp.setLhs((ExpressionOperator) logToPhyMap.get(op.getLhs()));
    exprOp.setRhs((ExpressionOperator) logToPhyMap.get(op.getRhs()));
    OperatorPlan oPlan = op.getPlan();

    currentPlan.add(exprOp);
    logToPhyMap.put(op, exprOp);

    List<Operator> successors = oPlan.getSuccessors(op);
    if (successors == null) {
        return;
    }
    for (Operator lo : successors) {
        PhysicalOperator from = logToPhyMap.get(lo);
        try {
            currentPlan.connect(from, exprOp);
        } catch (PlanException e) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:30,
代码来源:ExpToPhyTranslationVisitor.java

示例22: physicalToRDD

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void physicalToRDD(PhysicalPlan plan,
        PhysicalOperator physicalOperator,
        Map<OperatorKey, RDD<Tuple>> rdds,
        Map<Class<? extends PhysicalOperator>, POConverter> convertMap)
        throws IOException {

    RDD<Tuple> nextRDD = null;
    List<PhysicalOperator> predecessors = plan
            .getPredecessors(physicalOperator);
    List<RDD<Tuple>> predecessorRdds = Lists.newArrayList();
    if (predecessors != null) {
        for (PhysicalOperator predecessor : predecessors) {
            physicalToRDD(plan, predecessor, rdds, convertMap);
            predecessorRdds.add(rdds.get(predecessor.getOperatorKey()));
        }
    }

    POConverter converter = convertMap.get(physicalOperator.getClass());
    if (converter == null) {
        throw new IllegalArgumentException(
                "Spork unsupported PhysicalOperator: " + physicalOperator);
    }

    LOG.info("Converting operator "
            + physicalOperator.getClass().getSimpleName() + " "
            + physicalOperator);
    nextRDD = converter.convert(predecessorRdds, physicalOperator);

    if (POStore.class.equals(physicalOperator.getClass())) {
        return;
    }

    if (nextRDD == null) {
        throw new IllegalArgumentException(
                "RDD should not be null after PhysicalOperator: "
                        + physicalOperator);
    }

    rdds.put(physicalOperator.getOperatorKey(), nextRDD);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:41,
代码来源:SparkLauncher.java

示例23: reduce

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
/**
 * The reduce function which packages the key and List &lt;Tuple&gt;
 * into key, Bag&lt;Tuple&gt; after converting Hadoop type key into Pig type.
 * The package result is either collected as is, if the reduce plan is
 * empty or after passing through the reduce plan.
 */
@Override
protected void reduce(PigNullableWritable key, Iterable<NullableTuple> tupIter, Context context)
        throws IOException, InterruptedException {
    if(!initialized) {
        initialized = true;
        pigReporter.setRep(context);
        PhysicalOperator.setReporter(pigReporter);

        boolean aggregateWarning = "true".equalsIgnoreCase(pigContext.getProperties().getProperty("aggregate.warning"));
        PigStatusReporter pigStatusReporter = PigStatusReporter.getInstance();
        pigStatusReporter.setContext(new MRTaskContext(context));
        PigHadoopLogger pigHadoopLogger = PigHadoopLogger.getInstance();
        pigHadoopLogger.setReporter(pigStatusReporter);
        pigHadoopLogger.setAggregate(aggregateWarning);
        PhysicalOperator.setPigLogger(pigHadoopLogger);
    }

    // In the case we optimize, we combine
    // POPackage and POForeach - so we could get many
    // tuples out of the getnext() call of POJoinPackage
    // In this case, we process till we see EOP from
    // POJoinPacakage.getNext()
    if (pack.getPkgr() instanceof JoinPackager)
    {
        pack.attachInput(key, tupIter.iterator());
        while (true)
        {
            if (processOnePackageOutput(context))
                break;
        }
    }
    else {
        // not optimized, so package will
        // give only one tuple out for the key
        pack.attachInput(key, tupIter.iterator());
        processOnePackageOutput(context);
    }

}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:46,
代码来源:PigCombiner.java

示例24: getLogToDataMap

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void getLogToDataMap(Map<PhysicalOperator, DataBag> phyToDataMap) {
    logToDataMap.clear();
    for (Operator lo : logToPhyMap.keySet()) {
        if (logToPhyMap.get(lo) != null)
            logToDataMap.put(lo, phyToDataMap.get(logToPhyMap.get(lo)));
    }
    
    // set the LO-to-Data mapping for the ForEach inner plans
    for (Map.Entry<LOForEach, Map<LogicalRelationalOperator, DataBag>> entry : forEachInnerLogToDataMap.entrySet()) {
        entry.getValue().clear();
        for (Map.Entry<LogicalRelationalOperator, PhysicalOperator>  innerEntry : forEachInnerLogToPhyMap.get(entry.getKey()).entrySet()) {
            entry.getValue().put(innerEntry.getKey(), phyToDataMap.get(innerEntry.getValue()));
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:16,
代码来源:ExampleGenerator.java

示例25: attachBinaryExpressionOperator

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void attachBinaryExpressionOperator( BinaryExpression op, 
        BinaryExpressionOperator exprOp ) throws FrontendException {
    // We dont have aliases in ExpressionOperators
    // exprOp.setAlias(op.getAlias());
    
    
    exprOp.setResultType(op.getLhs().getType());
    exprOp.setLhs((ExpressionOperator) logToPhyMap.get(op.getLhs()));
    exprOp.setRhs((ExpressionOperator) logToPhyMap.get(op.getRhs()));
    OperatorPlan oPlan = op.getPlan();

    currentPlan.add(exprOp);
    logToPhyMap.put(op, exprOp);

    List<Operator> successors = oPlan.getSuccessors(op);
    if (successors == null) {
        return;
    }
    for (Operator lo : successors) {
        PhysicalOperator from = logToPhyMap.get(lo);
        try {
            currentPlan.connect(from, exprOp);
        } catch (PlanException e) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:30,
代码来源:ExpToPhyTranslationVisitor.java

示例26: testProcessInput

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@Test
public void testProcessInput() throws ExecException {
    // Stand-alone tests
    Result res = op.processInput();
    assertEquals(POStatus.STATUS_EOP, res.returnStatus);
    op.attachInput(t);
    res = op.processInput();
    assertEquals(POStatus.STATUS_OK, res.returnStatus);
    assertEquals(t, res.result);
    op.detachInput();
    res = op.processInput();
    assertEquals(POStatus.STATUS_EOP, res.returnStatus);

    // With input operator
    List<PhysicalOperator> inp = new ArrayList<PhysicalOperator>();
    inp.add(inpOp);
    op.setInputs(inp);
    op.processInput();
    assertEquals(POStatus.STATUS_EOP, res.returnStatus);

    inpOp.attachInput(t);
    res = op.processInput();
    assertEquals(POStatus.STATUS_OK, res.returnStatus);
    assertEquals(t, res.result);
    inpOp.detachInput();
    res = op.processInput();
    assertEquals(POStatus.STATUS_EOP, res.returnStatus);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:29,
代码来源:TestPhyOp.java

示例27: visitCounter

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
/**
 * For the counter job, it depends if it is row number or not.
 * In case of being a row number, any previous jobs are saved
 * and POCounter is added as a leaf on a map task.
 * If it is not, then POCounter is added as a leaf on a reduce
 * task (last sorting phase).
 **/
@Override
public void visitCounter(POCounter op) throws VisitorException {
    try{
        if(op.isRowNumber()) {
            List<PhysicalOperator> mpLeaves = curMROp.mapPlan.getLeaves();
            PhysicalOperator leaf = mpLeaves.get(0);
            if ( !curMROp.isMapDone() && !curMROp.isRankOperation() )
            {
                curMROp.mapPlan.addAsLeaf(op);
            } else {
                FileSpec fSpec = getTempFileSpec();
                MapReduceOper prevMROper = endSingleInputPlanWithStr(fSpec);
                MapReduceOper mrCounter = startNew(fSpec, prevMROper);
                mrCounter.mapPlan.addAsLeaf(op);
                curMROp = mrCounter;
            }
        } else {
            curMROp.reducePlan.addAsLeaf(op);
        }

        phyToMROpMap.put(op, curMROp);
    }catch(Exception e){
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:35,
代码来源:MRCompiler.java

示例28: revisit

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
/**
 * revisit an enhanced physical plan from MR compilation
 * @param plan a physical plan to be traversed
 */
public void revisit(PhysicalPlan plan) throws VisitorException {
    pushWalker(new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
    revisit = true;
    PhysicalPlan oriPlan = mPlan;
    mPlan = plan;
    visit();
    mPlan = oriPlan;
    popWalker();
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:14,
代码来源:IllustratorAttacher.java

示例29: phyToMRTransform

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void phyToMRTransform(PhysicalPlan plan, Map<PhysicalOperator, DataBag> phyToDataMap) {
    // remap the LO to PO as result of the MR compilation may have changed PO in the MR plans
    Map<PhysicalOperator, PhysicalOperator> phyToMRMap = localMRRunner.getPhyToMRMap();
    for (Map.Entry<PhysicalOperator, Operator> entry : poToLogMap.entrySet()) {
        if (phyToMRMap.get(entry.getKey()) != null) {
            PhysicalOperator poInMR = phyToMRMap.get(entry.getKey());
            logToDataMap.put(entry.getValue(), phyToDataMap.get(poInMR));
            poToEqclassesMap.put(entry.getKey(), poToEqclassesMap.get(poInMR));
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:12,
代码来源:ExampleGenerator.java

示例30: depthFirstPP

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
protected void depthFirstPP(Element parentNode) throws VisitorException {
    List<PhysicalOperator> leaves = mPlan.getLeaves();
    Collections.sort(leaves);
    for (PhysicalOperator leaf : leaves) {
        depthFirst(leaf, parentNode);
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork,
代码行数:8,
代码来源:XMLPhysicalPlanPrinter.java

示例31: isDiamondMROper

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private boolean isDiamondMROper(MapReduceOper mr) {
    
    // We'll remove this mr as part of diamond query optimization
    // only if this mr is a trivial one, that is, it's plan
    // has either two operators (load followed by store) or three operators 
    // (the operator between the load and store must be a foreach,
    // introduced by casting operation).
    // 
    // We won't optimize in other cases where there're more operators
    // in the plan. Otherwise those operators world run multiple times 
    // in the successor MR operators which may not give better
    // performance.
    boolean rtn = false;
    if (isMapOnly(mr)) {
        PhysicalPlan pl = mr.mapPlan;
        if (pl.size() == 2 || pl.size() == 3) {               
            PhysicalOperator root = pl.getRoots().get(0);
            PhysicalOperator leaf = pl.getLeaves().get(0);
            if (root instanceof POLoad && leaf instanceof POStore) {
                if (pl.size() == 3) {
                    PhysicalOperator mid = pl.getSuccessors(root).get(0);
                    if (mid instanceof POForEach) {
                        rtn = true;
                    }                      
                } else {
                    rtn = true;
                }
            }
        }
    }
    return rtn;
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:33,
代码来源:MultiQueryOptimizer.java

示例32: assertPredecessorSizeGreaterThan

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
public static void assertPredecessorSizeGreaterThan(List<JavaDStream<Tuple>> predecessors,
                                         PhysicalOperator physicalOperator, int size) {
    if (predecessors.size() <= size) {
        throw new RuntimeException("Should have greater than" + size + " predecessors for " +
                physicalOperator.getClass() + ". Got : " + predecessors.size());
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:8,
代码来源:SparkUtil.java

示例33: scan

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void scan(MapReduceOper mr, PhysicalOperator op, String fileName) {
  	
if (op instanceof POUserFunc) {
	if (((POUserFunc)op).getFuncSpec().getClassName().equals(
			"org.apache.pig.impl.builtin.PartitionSkewedKeys")) {
		
		String[] ctorArgs = ((POUserFunc)op).getFuncSpec().getCtorArgs();
		ctorArgs[2] = fileName;    		
		return;
	}
}else if (op instanceof POForEach) {
	List<PhysicalPlan> pl = ((POForEach)op).getInputPlans();
	for(PhysicalPlan plan: pl) {
		List<PhysicalOperator> list = plan.getLeaves();
		for (PhysicalOperator pp: list) {
			scan(mr, pp, fileName);
		}
	}
}else{
	List<PhysicalOperator> preds = mr.reducePlan.getPredecessors(op);
   	
   	if (preds == null) {
   		return;
   	}
   	
   	for(PhysicalOperator p: preds) {	    		    	
   		scan(mr, p, fileName);	    		
   	}
}
  }
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:31,
代码来源:SampleOptimizer.java

示例34: poSortDescInt

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
public void poSortDescInt(DataBag input) throws ExecException {
    List<PhysicalPlan> sortPlans = new LinkedList<PhysicalPlan>();
    POProject pr1 = new POProject(new OperatorKey("", r.nextLong()), -1, 1);
    pr1.setResultType(DataType.INTEGER);
    PhysicalPlan expPlan = new PhysicalPlan();
    expPlan.add(pr1);
    sortPlans.add(expPlan);
    List<Boolean> mAscCols = new LinkedList<Boolean>();
    mAscCols.add(false);
    PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
    List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
    inputs.add(read);
    POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
            sortPlans, mAscCols, null);
    Tuple t = null;
    Result res1 = sort.getNextTuple();
    // System.out.println(res1.result);
    Result res2 = sort.getNextTuple();
    while (res2.returnStatus != POStatus.STATUS_EOP) {
        Object i1 = ((Tuple) res1.result).get(1);
        Object i2 = ((Tuple) res2.result).get(1);
        int i = DataType.compare(i1, i2);
        assertEquals(true, (i >= 0));
        // System.out.println(res2.result);
        res1 = res2;
        res2 = sort.getNextTuple();
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:29,
代码来源:TestPOSort.java

示例35: setUp1

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
private void setUp1() throws PlanException, ExecException{
    lr = GenPhyOp.topLocalRearrangeOPWithPlanPlain(0,0,db.iterator().next());
    POProject proj = GenPhyOp.exprProject();
    proj.setColumn(0);
    proj.setResultType(DataType.TUPLE);
    proj.setOverloaded(true);
    Tuple t = new DefaultTuple();
    t.append(db);
    proj.attachInput(t);
    List<PhysicalOperator> inputs = new ArrayList<PhysicalOperator>();
    inputs.add(proj);
    lr.setInputs(inputs);
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:14,
代码来源:TestLocalRearrange.java

示例36: setAccumStart

点赞 2

import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; //导入依赖的package包/类
@Override
public void setAccumStart() {
    super.setAccumStart();
    for(PhysicalPlan p : inputPlans) {
        Iterator<PhysicalOperator> iter = p.iterator();
        while(iter.hasNext()) {
            PhysicalOperator po = iter.next();
            if (po instanceof ExpressionOperator || po instanceof PODistinct) {
                po.setAccumStart();
            }
        }
    }
}
 

开发者ID:sigmoidanalytics,
项目名称:spork-streaming,
代码行数:14,
代码来源:POForEach.java


版权声明:本文转自网络文章,转载此文章仅为分享知识,如有侵权,请联系管理员进行删除。
喜欢 (0)