1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.wal;
19
20 import org.apache.commons.logging.Log;
21 import org.apache.commons.logging.LogFactory;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.fs.FileStatus;
24 import org.apache.hadoop.fs.FileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.TableName;
28 import org.apache.hadoop.hbase.HTableDescriptor;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.KeyValue;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.fs.HFileSystem;
34 import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl;
35 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
36 import org.apache.hadoop.hbase.testclassification.MediumTests;
37 import org.apache.hadoop.hbase.util.Bytes;
38 import org.apache.hadoop.hbase.util.FSUtils;
39 import org.junit.AfterClass;
40 import org.junit.Before;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44
45 import java.io.IOException;
46 import java.util.ArrayList;
47 import java.util.Collections;
48 import java.util.List;
49 import java.util.concurrent.atomic.AtomicLong;
50
51 import static org.junit.Assert.assertEquals;
52
53 @Category(MediumTests.class)
54 public class TestWALRootDir {
55 private static final Log LOG = LogFactory.getLog(TestWALRootDir.class);
56 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57 private static Configuration conf;
58 private static FileSystem fs;
59 private static FileSystem walFs;
60 static final TableName tableName = TableName.valueOf("TestWALWALDir");
61 private static final byte [] rowName = Bytes.toBytes("row");
62 private static final byte [] family = Bytes.toBytes("column");
63 private static HTableDescriptor htd;
64 private static Path walRootDir;
65 private static Path rootDir;
66 private static WALFactory wals;
67
68 @Before
69 public void setUp() throws Exception {
70 cleanup();
71 }
72
73 @BeforeClass
74 public static void setUpBeforeClass() throws Exception {
75 conf = TEST_UTIL.getConfiguration();
76 TEST_UTIL.startMiniDFSCluster(1);
77 rootDir = TEST_UTIL.createRootDir();
78 walRootDir = TEST_UTIL.createWALRootDir();
79 fs = FSUtils.getRootDirFileSystem(conf);
80 walFs = FSUtils.getWALFileSystem(conf);
81 htd = new HTableDescriptor(tableName);
82 htd.addFamily(new HColumnDescriptor(family));
83 }
84
85 @AfterClass
86 public static void tearDownAfterClass() throws Exception {
87 cleanup();
88 TEST_UTIL.shutdownMiniDFSCluster();
89 }
90
91 @Test
92 public void testWALRootDir() throws Exception {
93 HRegionInfo regionInfo = new HRegionInfo(tableName);
94 wals = new WALFactory(conf, null, "testWALRootDir");
95 WAL log = wals.getWAL(regionInfo.getEncodedNameAsBytes()
96
97 );
98
99 assertEquals(1, getWALFiles(walFs, walRootDir).size());
100 byte [] value = Bytes.toBytes("value");
101 WALEdit edit = new WALEdit();
102 KeyValue kv = new KeyValue(rowName, family, Bytes.toBytes("1"),
103 System.currentTimeMillis(), value);
104 edit.add(kv);
105 AtomicLong al = new AtomicLong(kv.getSequenceId());
106 long txid = log.append(htd,regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 0), edit, al, true, null);
107 log.sync(txid);
108 assertEquals("Expect 1 log have been created", 1, getWALFiles(walFs, walRootDir).size());
109 log.rollWriter();
110
111 assertEquals(2, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size());
112 kv = new KeyValue(rowName, family, Bytes.toBytes("2"),
113 System.currentTimeMillis(), value);
114 edit.add(kv);
115 al.set(kv.getSequenceId());
116 txid = log.append(htd, regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 1), edit, al, true, null);
117 log.sync(txid);
118 log.rollWriter();
119 log.shutdown();
120
121 assertEquals("Expect 3 logs in WALs dir", 3, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size());
122 }
123
124 protected WALKey getWalKey(final long time, HRegionInfo hri, final long startPoint) {
125 MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
126 mvcc.initialize(startPoint);
127 return new WALKey(hri.getEncodedNameAsBytes(), tableName, time
128
129 );
130 }
131
132 private List<FileStatus> getWALFiles(FileSystem fs, Path dir)
133 throws IOException {
134 List<FileStatus> result = new ArrayList<FileStatus>();
135 LOG.debug("Scanning " + dir.toString() + " for WAL files");
136
137 FileStatus[] files = fs.listStatus(dir);
138 if (files == null) return Collections.emptyList();
139 for (FileStatus file : files) {
140 if (file.isDirectory()) {
141
142 result.addAll(getWALFiles(fs, file.getPath()));
143 } else {
144 String name = file.getPath().toString();
145 if (!name.startsWith(".")) {
146 result.add(file);
147 }
148 }
149 }
150 return result;
151 }
152
153 private static void cleanup() throws Exception{
154 walFs.delete(walRootDir, true);
155 fs.delete(rootDir, true);
156 }
157
158 }
159