View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mob.mapreduce;
19  
20  import static org.junit.Assert.assertArrayEquals;
21  import static org.junit.Assert.assertEquals;
22  
23  import java.io.IOException;
24  import java.util.List;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.CommonConfigurationKeys;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.mob.MobConstants;
32  import org.apache.hadoop.hbase.mob.MobUtils;
33  import org.apache.hadoop.hbase.testclassification.MediumTests;
34  import org.apache.hadoop.io.IOUtils;
35  import org.apache.hadoop.io.SequenceFile;
36  import org.apache.hadoop.io.serializer.JavaSerialization;
37  import org.apache.hadoop.io.serializer.WritableSerialization;
38  import org.junit.AfterClass;
39  import org.junit.BeforeClass;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  @Category(MediumTests.class)
44  public class TestMobSweepJob {
45  
46    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
47  
48    @BeforeClass
49    public static void setUpBeforeClass() throws Exception {
50      TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
51      TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
52      TEST_UTIL.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
53          JavaSerialization.class.getName() + "," + WritableSerialization.class.getName());
54      TEST_UTIL.startMiniCluster();
55    }
56  
57    @AfterClass
58    public static void tearDownAfterClass() throws Exception {
59      TEST_UTIL.shutdownMiniCluster();
60    }
61  
62    private void writeFileNames(FileSystem fs, Configuration conf, Path path,
63        String[] filesNames) throws IOException {
64      // write the names to a sequence file
65      SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path,
66          String.class, String.class);
67      try {
68        for (String fileName : filesNames) {
69          writer.append(fileName, MobConstants.EMPTY_STRING);
70        }
71      } finally {
72        IOUtils.closeStream(writer);
73      }
74    }
75  
76    @Test
77    public void testSweeperJobWithOutUnusedFile() throws Exception {
78      FileSystem fs = TEST_UTIL.getTestFileSystem();
79      Configuration configuration = new Configuration(
80          TEST_UTIL.getConfiguration());
81      Path vistiedFileNamesPath = new Path(MobUtils.getMobHome(configuration),
82          "/hbase/mobcompaction/SweepJob/working/names/0/visited");
83      Path allFileNamesPath = new Path(MobUtils.getMobHome(configuration),
84          "/hbase/mobcompaction/SweepJob/working/names/0/all");
85      configuration.set(SweepJob.WORKING_VISITED_DIR_KEY,
86          vistiedFileNamesPath.toString());
87      configuration.set(SweepJob.WORKING_ALLNAMES_FILE_KEY,
88          allFileNamesPath.toString());
89  
90      writeFileNames(fs, configuration, allFileNamesPath, new String[] { "1",
91          "2", "3", "4", "5", "6"});
92  
93      Path r0 = new Path(vistiedFileNamesPath, "r0");
94      writeFileNames(fs, configuration, r0, new String[] { "1",
95          "2", "3"});
96      Path r1 = new Path(vistiedFileNamesPath, "r1");
97      writeFileNames(fs, configuration, r1, new String[] { "1", "4", "5"});
98      Path r2 = new Path(vistiedFileNamesPath, "r2");
99      writeFileNames(fs, configuration, r2, new String[] { "2", "3", "6"});
100 
101     SweepJob sweepJob = new SweepJob(configuration, fs);
102     List<String> toBeArchived = sweepJob.getUnusedFiles(configuration);
103 
104     assertEquals(0, toBeArchived.size());
105   }
106 
107   @Test
108   public void testSweeperJobWithUnusedFile() throws Exception {
109     FileSystem fs = TEST_UTIL.getTestFileSystem();
110     Configuration configuration = new Configuration(
111         TEST_UTIL.getConfiguration());
112     Path vistiedFileNamesPath = new Path(MobUtils.getMobHome(configuration),
113         "/hbase/mobcompaction/SweepJob/working/names/1/visited");
114     Path allFileNamesPath = new Path(MobUtils.getMobHome(configuration),
115         "/hbase/mobcompaction/SweepJob/working/names/1/all");
116     configuration.set(SweepJob.WORKING_VISITED_DIR_KEY,
117         vistiedFileNamesPath.toString());
118     configuration.set(SweepJob.WORKING_ALLNAMES_FILE_KEY,
119         allFileNamesPath.toString());
120 
121     writeFileNames(fs, configuration, allFileNamesPath, new String[] { "1",
122         "2", "3", "4", "5", "6"});
123 
124     Path r0 = new Path(vistiedFileNamesPath, "r0");
125     writeFileNames(fs, configuration, r0, new String[] { "1",
126         "2", "3"});
127     Path r1 = new Path(vistiedFileNamesPath, "r1");
128     writeFileNames(fs, configuration, r1, new String[] { "1", "5"});
129     Path r2 = new Path(vistiedFileNamesPath, "r2");
130     writeFileNames(fs, configuration, r2, new String[] { "2", "3"});
131 
132     SweepJob sweepJob = new SweepJob(configuration, fs);
133     List<String> toBeArchived = sweepJob.getUnusedFiles(configuration);
134 
135     assertEquals(2, toBeArchived.size());
136     assertArrayEquals(new String[]{"4", "6"}, toBeArchived.toArray(new String[0]));
137   }
138 
139   @Test
140   public void testSweeperJobWithRedundantFile() throws Exception {
141     FileSystem fs = TEST_UTIL.getTestFileSystem();
142     Configuration configuration = new Configuration(
143         TEST_UTIL.getConfiguration());
144     Path vistiedFileNamesPath = new Path(MobUtils.getMobHome(configuration),
145         "/hbase/mobcompaction/SweepJob/working/names/2/visited");
146     Path allFileNamesPath = new Path(MobUtils.getMobHome(configuration),
147         "/hbase/mobcompaction/SweepJob/working/names/2/all");
148     configuration.set(SweepJob.WORKING_VISITED_DIR_KEY,
149         vistiedFileNamesPath.toString());
150     configuration.set(SweepJob.WORKING_ALLNAMES_FILE_KEY,
151         allFileNamesPath.toString());
152 
153     writeFileNames(fs, configuration, allFileNamesPath, new String[] { "1",
154         "2", "3", "4", "5", "6"});
155 
156     Path r0 = new Path(vistiedFileNamesPath, "r0");
157     writeFileNames(fs, configuration, r0, new String[] { "1",
158         "2", "3"});
159     Path r1 = new Path(vistiedFileNamesPath, "r1");
160     writeFileNames(fs, configuration, r1, new String[] { "1", "5", "6", "7"});
161     Path r2 = new Path(vistiedFileNamesPath, "r2");
162     writeFileNames(fs, configuration, r2, new String[] { "2", "3", "4"});
163 
164     SweepJob sweepJob = new SweepJob(configuration, fs);
165     List<String> toBeArchived = sweepJob.getUnusedFiles(configuration);
166 
167     assertEquals(0, toBeArchived.size());
168   }
169 }