Java重写MapReduce

/*** 统计单词个数*/​
package com.shujia.mr;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;public class MR01 {/*** map阶段 *     java 1*     hadoop1*     ...*/public static class WordMapper extends Mapper{@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String line =value.toString();//相对每行数据处理，Text转化为Stringint v=1;context.write(new Text(line),new LongWritable(v));}}/*** reduce阶段 *     聚合*/public static class WordReducer extends Reducer{@Overrideprotected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {int count=0;for (LongWritable value : values) {count+=value.get();//.get()是将LongWritable转为int类型}context.write(key,new LongWritable(count));}}public static void main(String[] args) throws Exception{//配置mapreduceJob job = Job.getInstance();job.setJobName("第一个mr程序 单词统计");job.setJarByClass(MR01.class);//map段所在类的位置job.setMapperClass(WordMapper.class);//reduce段所在类的位置job.setReducerClass(WordReducer.class);//指定map段kv的输出类型job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(LongWritable.class);//指定reduce段kv的输出类型job.setOutputKeyClass(Text.class);job.setOutputValueClass(LongWritable.class);//指定路径Path input = new Path("/words.txt");Path out = new Path("/output");//输出路径不能已存在，手动加上 已存在删除FileSystem fs = FileSystem.get(new Configuration());if(fs.exists(out)){fs.delete(out,true);}FileInputFormat.addInputPath(job,input);FileOutputFormat.setOutputPath(job,out);//启动job.waitForCompletion(true);System.out.println("正在运行mr");}
}​

生成包jar

hadoop jar hadoop-1.0-SNAPSHOT.jar com.shujia.mr.MR01

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！