HADOOP笔试(3)

126. + r.items[5] + \

127. + (r.stay_time / 1000) + \

128. + r.items[6] + \

129. rest.set(value);

130.

131.

132. 133.

134. 135.

136.

137.

138.

139. 140. 141. 142. }复制代码

context.write(key, rest);

}

}

static class Record {

String[] items;

long start_time;

long stay_time;

}

}

2.

2.1 使用 find + sed 来实现：

find /home/ocetl/app/hadoop -exec sed -i 's/\\$HADOOP_HOME\\$/\\/home\\/ocetl\\/app\\/hadoop/g' {} \\; 2.2 直接使用ssh的参数

1. #!/bin/bash

2. if [ $# -ne 1 ]

3. then

4. echo \

5. exit

6. fi

7.

8. for i in H1 H2 H3 H4 H5 H6 H7 H8 H9 H10

9. do

10. echo \

11. ssh $i \

12. done

复制代码

3.

3.1 不了解，HDFS用了缓存

3.2 问题是当硬盘空间很大，而内存页面缓存很少的时候，DN的Block report需要很长时间生成，而此时 FSVolumeSet 锁是锁住的，因此所有读写操作都无法执行，最终导致那些操作超时。此问题是建议提供一种方法使block report不需要持有FSVolumeSet锁，从而不会导致那些任务失败。

4. 只是替换分隔符从空格到逗号，以及增加搜索关键字列表：

1. import org.apache.hadoop.conf.Configuration;

2. import org.apache.hadoop.fs.Path;

3. import org.apache.hadoop.io.IntWritable;

4. import org.apache.hadoop.io.LongWritable;

5. import org.apache.hadoop.io.Text;

6. import org.apache.hadoop.mapreduce.Job;

7. import org.apache.hadoop.mapreduce.Mapper;

8. import org.apache.hadoop.mapreduce.Reducer;

9. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

10. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

11. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

12. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

13.

14. import java.io.IOException;

15. import java.util.ArrayList;

16.

17. public class WordCount {

18.

19. public static class Map extends Mapper {

20. private final static IntWritable one = new IntWritable(1);

21. private Text word = new Text();

22. private final static ArrayList target_words = new ArrayList();

23.

24. public void map(LongWritable key, Text value, Context context) throws IOException,

InterruptedException {

25. String[] items = value.toString().toLowerCase().replaceAll(\

\

26. for (String item : items) {

27. if (target_words.contains(item)) {

28. word.set(item);

29. context.write(word, one);

30. }

31. }

32. }

33.

34. public static void clear() {

35. target_words.clear();

36. }

37.

38. public static void add(String word) {

39. target_words.add(word);

40. }

41. }

42.

43. public static class Reduce extends Reducer {

44.

45. public void reduce(Text key, Iterable values, Context context)

46. throws IOException, InterruptedException {

47. int sum = 0;

48. for (IntWritable val : values) {

49. sum += val.get();