1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mob.mapreduce;
19
20 import static org.junit.Assert.assertArrayEquals;
21 import static org.junit.Assert.assertEquals;
22
23 import java.io.IOException;
24 import java.util.List;
25
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.CommonConfigurationKeys;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.mob.MobConstants;
32 import org.apache.hadoop.hbase.mob.MobUtils;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.io.IOUtils;
35 import org.apache.hadoop.io.SequenceFile;
36 import org.apache.hadoop.io.serializer.JavaSerialization;
37 import org.apache.hadoop.io.serializer.WritableSerialization;
38 import org.junit.AfterClass;
39 import org.junit.BeforeClass;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42
43 @Category(MediumTests.class)
44 public class TestMobSweepJob {
45
46 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
47
48 @BeforeClass
49 public static void setUpBeforeClass() throws Exception {
50 TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
51 TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
52 TEST_UTIL.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
53 JavaSerialization.class.getName() + "," + WritableSerialization.class.getName());
54 TEST_UTIL.startMiniCluster();
55 }
56
57 @AfterClass
58 public static void tearDownAfterClass() throws Exception {
59 TEST_UTIL.shutdownMiniCluster();
60 }
61
62 private void writeFileNames(FileSystem fs, Configuration conf, Path path,
63 String[] filesNames) throws IOException {
64
65 SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path,
66 String.class, String.class);
67 try {
68 for (String fileName : filesNames) {
69 writer.append(fileName, MobConstants.EMPTY_STRING);
70 }
71 } finally {
72 IOUtils.closeStream(writer);
73 }
74 }
75
76 @Test
77 public void testSweeperJobWithOutUnusedFile() throws Exception {
78 FileSystem fs = TEST_UTIL.getTestFileSystem();
79 Configuration configuration = new Configuration(
80 TEST_UTIL.getConfiguration());
81 Path vistiedFileNamesPath = new Path(MobUtils.getMobHome(configuration),
82 "/hbase/mobcompaction/SweepJob/working/names/0/visited");
83 Path allFileNamesPath = new Path(MobUtils.getMobHome(configuration),
84 "/hbase/mobcompaction/SweepJob/working/names/0/all");
85 configuration.set(SweepJob.WORKING_VISITED_DIR_KEY,
86 vistiedFileNamesPath.toString());
87 configuration.set(SweepJob.WORKING_ALLNAMES_FILE_KEY,
88 allFileNamesPath.toString());
89
90 writeFileNames(fs, configuration, allFileNamesPath, new String[] { "1",
91 "2", "3", "4", "5", "6"});
92
93 Path r0 = new Path(vistiedFileNamesPath, "r0");
94 writeFileNames(fs, configuration, r0, new String[] { "1",
95 "2", "3"});
96 Path r1 = new Path(vistiedFileNamesPath, "r1");
97 writeFileNames(fs, configuration, r1, new String[] { "1", "4", "5"});
98 Path r2 = new Path(vistiedFileNamesPath, "r2");
99 writeFileNames(fs, configuration, r2, new String[] { "2", "3", "6"});
100
101 SweepJob sweepJob = new SweepJob(configuration, fs);
102 List<String> toBeArchived = sweepJob.getUnusedFiles(configuration);
103
104 assertEquals(0, toBeArchived.size());
105 }
106
107 @Test
108 public void testSweeperJobWithUnusedFile() throws Exception {
109 FileSystem fs = TEST_UTIL.getTestFileSystem();
110 Configuration configuration = new Configuration(
111 TEST_UTIL.getConfiguration());
112 Path vistiedFileNamesPath = new Path(MobUtils.getMobHome(configuration),
113 "/hbase/mobcompaction/SweepJob/working/names/1/visited");
114 Path allFileNamesPath = new Path(MobUtils.getMobHome(configuration),
115 "/hbase/mobcompaction/SweepJob/working/names/1/all");
116 configuration.set(SweepJob.WORKING_VISITED_DIR_KEY,
117 vistiedFileNamesPath.toString());
118 configuration.set(SweepJob.WORKING_ALLNAMES_FILE_KEY,
119 allFileNamesPath.toString());
120
121 writeFileNames(fs, configuration, allFileNamesPath, new String[] { "1",
122 "2", "3", "4", "5", "6"});
123
124 Path r0 = new Path(vistiedFileNamesPath, "r0");
125 writeFileNames(fs, configuration, r0, new String[] { "1",
126 "2", "3"});
127 Path r1 = new Path(vistiedFileNamesPath, "r1");
128 writeFileNames(fs, configuration, r1, new String[] { "1", "5"});
129 Path r2 = new Path(vistiedFileNamesPath, "r2");
130 writeFileNames(fs, configuration, r2, new String[] { "2", "3"});
131
132 SweepJob sweepJob = new SweepJob(configuration, fs);
133 List<String> toBeArchived = sweepJob.getUnusedFiles(configuration);
134
135 assertEquals(2, toBeArchived.size());
136 assertArrayEquals(new String[]{"4", "6"}, toBeArchived.toArray(new String[0]));
137 }
138
139 @Test
140 public void testSweeperJobWithRedundantFile() throws Exception {
141 FileSystem fs = TEST_UTIL.getTestFileSystem();
142 Configuration configuration = new Configuration(
143 TEST_UTIL.getConfiguration());
144 Path vistiedFileNamesPath = new Path(MobUtils.getMobHome(configuration),
145 "/hbase/mobcompaction/SweepJob/working/names/2/visited");
146 Path allFileNamesPath = new Path(MobUtils.getMobHome(configuration),
147 "/hbase/mobcompaction/SweepJob/working/names/2/all");
148 configuration.set(SweepJob.WORKING_VISITED_DIR_KEY,
149 vistiedFileNamesPath.toString());
150 configuration.set(SweepJob.WORKING_ALLNAMES_FILE_KEY,
151 allFileNamesPath.toString());
152
153 writeFileNames(fs, configuration, allFileNamesPath, new String[] { "1",
154 "2", "3", "4", "5", "6"});
155
156 Path r0 = new Path(vistiedFileNamesPath, "r0");
157 writeFileNames(fs, configuration, r0, new String[] { "1",
158 "2", "3"});
159 Path r1 = new Path(vistiedFileNamesPath, "r1");
160 writeFileNames(fs, configuration, r1, new String[] { "1", "5", "6", "7"});
161 Path r2 = new Path(vistiedFileNamesPath, "r2");
162 writeFileNames(fs, configuration, r2, new String[] { "2", "3", "4"});
163
164 SweepJob sweepJob = new SweepJob(configuration, fs);
165 List<String> toBeArchived = sweepJob.getUnusedFiles(configuration);
166
167 assertEquals(0, toBeArchived.size());
168 }
169 }