向HDFS上传本地文件
|
1
|
|
2
3
4
5
6
7
8
9
|
public static void uploadInputFile(String localFile) throws IOException{ Configuration conf = new Configuration(); String hdfsPath = "hdfs://localhost:9000/"; String hdfsInput = "hdfs://localhost:9000/user/hadoop/input"; FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.copyFromLocalFile(new Path(localFile), new Path(hdfsInput)); fs.close(); System.out.println("已经上传文件到input文件夹啦"); } |
将output文件下载到本地
|
1
|
|
2
3
4
5
6
7
8
9
10
|
public static void getOutput(String outputfile) throws IOException{ String remoteFile = "hdfs://localhost:9000/user/hadoop/output/part-r-00000"; Path path = new Path(remoteFile); Configuration conf = new Configuration(); String hdfsPath = "hdfs://localhost:9000/"; FileSystem fs = FileSystem.get(URI.create(hdfsPath),conf); fs.copyToLocalFile(path, new Path(outputfile)); System.out.println("已经将输出文件保留到本地文件"); fs.close(); } |
删除hdfs中的文件
|
1
|
|
2
3
4
5
6
7
8
9
10
|
public static void deleteOutput() throws IOException{ Configuration conf = new Configuration(); String hdfsOutput = "hdfs://localhost:9000/user/hadoop/output"; String hdfsPath = "hdfs://localhost:9000/"; Path path = new Path(hdfsOutput); FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.deleteOnExit(path); fs.close(); System.out.println("output文件已经删除"); } |
执行mapReduce程序
创建Mapper类和Reducer类
|
1
|
|
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
|
public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{ private final static IntWritable one = new IntWritable(1); private Text word = new Text(); public void map(Object key, Text value, Context context) throws IOException, InterruptedException{ String line = value.toString(); line = line.replace("\\", ""); String regex = "性别：</span><span class=\"pt_detail\">(.*?)</span>"; Pattern pattern = Pattern.compile(regex); Matcher matcher = pattern.matcher(line); while(matcher.find()){ String term = matcher.group(1); word.set(term); context.write(word, one); } } } public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable>{ private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException{ int sum = 0; for(IntWritable val :values){ sum+= val.get(); } result.set(sum); context.write(key, result); } } |
执行mapReduce程序
|
1
|
|
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
public static void runMapReduce(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if(otherArgs.length != 2){ System.err.println("Usage: wordcount<in> <out>"); System.exit(2); } Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.out.println("mapReduce 执行完毕！"); System.exit(job.waitForCompletion(true)?0:1); } |
感谢阅读，希望能帮助到大家，谢谢大家对本站的支持！
原文链接：http://blog.csdn.net/qq_30843221/article/details/54429792
