資源簡介
一個(gè)MapReduce簡單程序示例 MapReduce hadoop

代碼片段和文件信息
package?cn.buaa.secondInvertindex;
import?java.io.IOException;
import?java.util.ArrayList;
import?java.util.StringTokenizer;
import?org.apache.hadoop.conf.Configuration;
import?org.apache.hadoop.fs.Path;
import?org.apache.hadoop.io.LongWritable;
import?org.apache.hadoop.io.Text;
import?org.apache.hadoop.mapreduce.Job;
import?org.apache.hadoop.mapreduce.Mapper;
import?org.apache.hadoop.mapreduce.Reducer;
import?org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import?org.apache.hadoop.mapreduce.lib.input.FileSplit;
import?org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import?org.apache.hadoop.util.GenericOptionsParser;
public?class?SencondInvertIndexWithCombiner?{
public?static?class?InvertIndexWithCombinerMap?extends?Mapper?{
private?static?final?String??TAG=“--InvertIndexWithCombinerMap--“;
private?Text?valueInfo=new?Text();
private?Text?keyInfo=new?Text();
????private?FileSplit?spliteName;
????
protected?void?map(LongWritable?key?Text?value
Context?context)
throws?IOException?InterruptedException?{
spliteName=(FileSplit)context.getInputSplit();
String?spliteFullName=spliteName.getPath().toString();
// System.out.println(spliteFullName);
int?beginIndex=spliteFullName.lastIndexOf(‘/‘);
// int?endIndex=spliteFullName.lastIndexOf(‘:‘);
// System.out.println(“beginloc:?“+beginIndex);
String?fileName=spliteName.getPath().toString().substring(beginIndex+1);
// ????System.out.println(fileName);
????
StringTokenizer?st=new?StringTokenizer(value.toString());
????????????while(st.hasMoreTokens()){
???????????? String?str=st.nextToken();
??????????//?? keyInfo.set(str+“:“+spliteName.getPath().toString());
????????????
???????????? //Test?demo
//??????????? if(“including“.equalsIgnoreCase(str)){
//???????????? System.out.println(“including?loc:?“+fileName);
//???????????? }
???????????? //end?of?Test?demo
???????????? keyInfo.set(str+“:“+fileName);
???????????? valueInfo.set(“1“);
???????????? context.write(keyInfo?valueInfo);
????????????}
????????????
}
}
public?static?class?InvertIndexWithCombinerConbiner?extends?Reducer{
private?int?interValue;
private?Text?keyInfo=new?Text();
private?Text?valueInfo=new?Text();
protected?void?reduce(Text?key?Iterable?value
Context?context)?
throws?IOException?InterruptedException?{
int?sum=0;
for?(Text?text?:?value)?{
?sum+=Integer.parseInt(text.toString());
}
???interValue=key.toString().indexOf(“:“);
???keyInfo.set(key.toString().substring(0?interValue));
???valueInfo.set(key.toString().substring(interValue+1)+“:“+sum);
???context.write(keyInfo?valueInfo);
}
}
/* public?static?class?InvertIndexWithCombinerPartition?extends?HashPartitioner{
@Override
public?int?getPartition(Text?key?Text?value?int?numReduceTasks)?{
//?TODO?Auto-generated?method?stub
return?super.getPartition(key?value
?屬性????????????大小?????日期????時(shí)間???名稱
-----------?---------??----------?-----??----
?????文件????1472000??2012-10-23?16:05??一個(gè)MapReduce簡單程序示例.ppt
?????文件???????8799??2012-10-22?12:46??SencondInvertIndexWithCombiner.java
-----------?---------??----------?-----??----
??????????????1480799????????????????????2
評論
共有 條評論