Hadoop学习记录（7）|Eclipse远程调试Hadoop

static class MyMapper extends
			Mapper<LongWritable, Text, Text, LongWritable> {

		@Override
		protected void map(LongWritable k1, Text v1, Context context)
				throws IOException, InterruptedException {
			// 对内容进行分词处理存到字符数组内
			StringTokenizer tokenizer = new StringTokenizer(v1.toString());
			// 创建Text k2
			Text k2 = new Text();
			// 遍历写入context中
			while (tokenizer.hasMoreTokens()) {
				k2.set(tokenizer.nextToken());
				context.write(k2, new LongWritable(1));
			}

		}

	}

static class MyReducer extends
			Reducer<Text, LongWritable, Text, LongWritable> {
		@Override
		protected void reduce(Text k2, Iterable<LongWritable> v2s,
				Context context) throws IOException, InterruptedException {
			long sum = 0;
			for(LongWritable val : v2s){
				sum += val.get();
			}
			context.write(k2, new LongWritable(sum));
		}
	}

public static void main(String[] args) throws Exception {
		
		if(args.length != 2){
			System.err.print("Usage:wordcount");
			System.exit(2);
		}
		
		Configuration conf = new Configuration();
		Job job = new Job(conf,WordCount.class.getSimpleName());
		//用eclipse插件运行相当于是jar包运行
		job.setJarByClass(WordCount.class);
		//设置mapper
		job.setMapperClass(MyMapper.class);
		//设置map输出k2的类型
		job.setMapOutputKeyClass(Text.class);
		//设置map输出v2的类型
		job.setMapOutputValueClass(LongWritable.class);
		//设置分区类
		job.setPartitionerClass(HashPartitioner.class);
		//设置作业数量
		job.setNumReduceTasks(1);
		//设置reducer类
		job.setReducerClass(MyReducer.class);
		//设置输出的格式
		job.setOutputFormatClass(TextOutputFormat.class);
		//设置k3的输出类型
		job.setOutputKeyClass(Text.class);
		//设置v3的输出类型
		job.setOutputValueClass(LongWritable.class);
		
		//这里是从外面传入参数
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		//提交任务,如果返回false代表有异常,使用system.exit结束java虚拟机,如果没问题返回0正常执行.
		System.exit(job.waitForCompletion(true)?0:1);
		
		
	}

14/03/11 15:58:22 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
14/03/11 15:58:22 ERROR security.UserGroupInformation: PriviledgedActionException as:Sky cause:java.io.IOException: Failed to set permissions of path: \tmp\hadoop-Sky\mapred\staging\Sky1823204560\.staging to 0700
Exception in thread "main" java.io.IOException: Failed to set permissions of path: \tmp\hadoop-Sky\mapred\staging\Sky1823204560\.staging to 0700
	at org.apache.hadoop.fs.FileUtil.checkReturnValue(FileUtil.java:689)
	at org.apache.hadoop.fs.FileUtil.setPermission(FileUtil.java:662)
	at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:509)
	at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:344)
	at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:189)
	at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:116)
	at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:918)
	at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:912)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:415)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
	at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:912)
	at org.apache.hadoop.mapreduce.Job.submit(Job.java:500)
	at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:530)
	at hdfs.WordCount.main(WordCount.java:58)