View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.wal;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.fs.FileStatus;
24  import org.apache.hadoop.fs.FileSystem;
25  import org.apache.hadoop.fs.Path;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.TableName;
28  import org.apache.hadoop.hbase.HTableDescriptor;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.KeyValue;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.fs.HFileSystem;
34  import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl;
35  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.util.FSUtils;
39  import org.junit.AfterClass;
40  import org.junit.Before;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  
45  import java.io.IOException;
46  import java.util.ArrayList;
47  import java.util.Collections;
48  import java.util.List;
49  import java.util.concurrent.atomic.AtomicLong;
50  
51  import static org.junit.Assert.assertEquals;
52  
53  @Category(MediumTests.class)
54  public class TestWALRootDir {
55    private static final Log LOG = LogFactory.getLog(TestWALRootDir.class);
56    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57    private static Configuration conf;
58    private static FileSystem fs;
59    private static FileSystem walFs;
60    static final TableName tableName = TableName.valueOf("TestWALWALDir");
61    private static final byte [] rowName = Bytes.toBytes("row");
62    private static final byte [] family = Bytes.toBytes("column");
63    private static HTableDescriptor htd;
64    private static Path walRootDir;
65    private static Path rootDir;
66    private static WALFactory wals;
67  
68    @Before
69    public void setUp() throws Exception {
70      cleanup();
71    }
72  
73    @BeforeClass
74    public static void setUpBeforeClass() throws Exception {
75      conf = TEST_UTIL.getConfiguration();
76      TEST_UTIL.startMiniDFSCluster(1);
77      rootDir = TEST_UTIL.createRootDir();
78      walRootDir = TEST_UTIL.createWALRootDir();
79      fs = FSUtils.getRootDirFileSystem(conf);
80      walFs = FSUtils.getWALFileSystem(conf);
81      htd = new HTableDescriptor(tableName);
82      htd.addFamily(new HColumnDescriptor(family));
83    }
84  
85    @AfterClass
86    public static void tearDownAfterClass() throws Exception {
87      cleanup();
88      TEST_UTIL.shutdownMiniDFSCluster();
89    }
90  
91    @Test
92    public void testWALRootDir() throws Exception {
93      HRegionInfo regionInfo = new HRegionInfo(tableName);
94      wals = new WALFactory(conf, null, "testWALRootDir");
95      WAL log = wals.getWAL(regionInfo.getEncodedNameAsBytes()
96          //, regionInfo.getTable().getNamespace()
97          );
98  
99      assertEquals(1, getWALFiles(walFs, walRootDir).size());
100     byte [] value = Bytes.toBytes("value");
101     WALEdit edit = new WALEdit();
102     KeyValue kv = new KeyValue(rowName, family, Bytes.toBytes("1"),
103         System.currentTimeMillis(), value);
104     edit.add(kv);
105     AtomicLong al = new AtomicLong(kv.getSequenceId());
106     long txid = log.append(htd,regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 0), edit, al, true, null);
107     log.sync(txid);
108     assertEquals("Expect 1 log have been created", 1, getWALFiles(walFs, walRootDir).size());
109     log.rollWriter();
110     //Create 1 more WAL
111     assertEquals(2, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size());
112     kv = new KeyValue(rowName, family, Bytes.toBytes("2"),
113         System.currentTimeMillis(), value);
114     edit.add(kv);
115     al.set(kv.getSequenceId());
116     txid = log.append(htd, regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 1), edit, al, true, null);
117     log.sync(txid);
118     log.rollWriter();
119     log.shutdown();
120 
121     assertEquals("Expect 3 logs in WALs dir", 3, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size());
122   }
123 
124   protected WALKey getWalKey(final long time, HRegionInfo hri, final long startPoint) {
125     MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
126     mvcc.initialize(startPoint);
127     return new WALKey(hri.getEncodedNameAsBytes(), tableName, time
128         // , mvcc
129         );
130   }
131 
132   private List<FileStatus> getWALFiles(FileSystem fs, Path dir)
133       throws IOException {
134     List<FileStatus> result = new ArrayList<FileStatus>();
135     LOG.debug("Scanning " + dir.toString() + " for WAL files");
136 
137     FileStatus[] files = fs.listStatus(dir);
138     if (files == null) return Collections.emptyList();
139     for (FileStatus file : files) {
140       if (file.isDirectory()) {
141         // recurse into sub directories
142         result.addAll(getWALFiles(fs, file.getPath()));
143       } else {
144         String name = file.getPath().toString();
145         if (!name.startsWith(".")) {
146           result.add(file);
147         }
148       }
149     }
150     return result;
151   }
152 
153   private static void cleanup() throws Exception{
154     walFs.delete(walRootDir, true);
155     fs.delete(rootDir, true);
156   }
157 
158 }
159