1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellComparator;
33 import org.apache.hadoop.hbase.CellScanner;
34 import org.apache.hadoop.hbase.CellUtil;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.client.Get;
42 import org.apache.hadoop.hbase.client.Result;
43 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
44 import org.apache.hadoop.hbase.testclassification.MediumTests;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.FSUtils;
47 import org.apache.hadoop.hbase.wal.WAL;
48 import org.apache.hadoop.hbase.wal.WALFactory;
49 import org.apache.hadoop.hbase.wal.WALKey;
50 import org.apache.hadoop.hbase.wal.WALSplitter;
51 import org.junit.Rule;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54 import org.junit.rules.TestName;
55
56
57
58
59 @Category({MediumTests.class})
60 public class TestRecoveredEdits {
61 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private static final Log LOG = LogFactory.getLog(TestRecoveredEdits.class);
63 @Rule public TestName testName = new TestName();
64
65
66
67
68
69
70
71
72 @Test (timeout=60000)
73 public void testReplayWorksThoughLotsOfFlushing() throws IOException {
74 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
75
76 conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
77
78
79 final String encodedRegionName = "4823016d8fca70b25503ee07f4c6d79f";
80 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(testName.getMethodName()));
81 final String columnFamily = "meta";
82 byte [][] columnFamilyAsByteArray = new byte [][] {Bytes.toBytes(columnFamily)};
83 htd.addFamily(new HColumnDescriptor(columnFamily));
84 HRegionInfo hri = new HRegionInfo(htd.getTableName()) {
85 @Override
86 public synchronized String getEncodedName() {
87 return encodedRegionName;
88 }
89
90
91 private byte [] encodedRegionNameAsBytes = null;
92 @Override
93 public synchronized byte[] getEncodedNameAsBytes() {
94 if (encodedRegionNameAsBytes == null) {
95 this.encodedRegionNameAsBytes = Bytes.toBytes(getEncodedName());
96 }
97 return this.encodedRegionNameAsBytes;
98 }
99 };
100 Path hbaseRootDir = TEST_UTIL.getDataTestDir();
101 FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
102 Path tableDir = FSUtils.getTableDir(hbaseRootDir, htd.getTableName());
103 HRegionFileSystem hrfs =
104 new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tableDir, hri);
105 if (fs.exists(hrfs.getRegionDir())) {
106 LOG.info("Region directory already exists. Deleting.");
107 fs.delete(hrfs.getRegionDir(), true);
108 }
109 HRegion region = HRegion.createHRegion(hri, hbaseRootDir, conf, htd, null);
110 assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
111 List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
112
113 assertTrue(storeFiles.isEmpty());
114 region.close();
115 Path regionDir = region.getRegionDir(hbaseRootDir, hri);
116 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);
117
118 Path recoveredEditsFile = new Path(new Path(
119 System.getProperty("project.build.testSourceDirectory", "src" + Path.SEPARATOR + "test"),
120 "data"), "0000000000000016310");
121
122 Path destination = new Path(recoveredEditsDir, recoveredEditsFile.getName());
123 fs.copyToLocalFile(recoveredEditsFile, destination);
124 assertTrue(fs.exists(destination));
125
126 region = HRegion.openHRegion(region, null);
127 assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
128 storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
129
130
131
132 assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10);
133
134 int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region);
135 LOG.info("Checked " + count + " edits made it in");
136 }
137
138
139
140
141
142
143
144
145
146 private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf,
147 final Path edits, final HRegion region)
148 throws IOException {
149 int count = 0;
150
151 WAL.Reader reader = null;
152 try {
153 reader = WALFactory.createReader(fs, edits, conf);
154 WAL.Entry entry;
155 while ((entry = reader.next()) != null) {
156 WALKey key = entry.getKey();
157 WALEdit val = entry.getEdit();
158 count++;
159
160 if (!Bytes.equals(key.getEncodedRegionName(),
161 region.getRegionInfo().getEncodedNameAsBytes())) {
162 continue;
163 }
164 Cell previous = null;
165 for (Cell cell: val.getCells()) {
166 if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) continue;
167 if (previous != null && CellComparator.compareRows(previous, cell) == 0) continue;
168 previous = cell;
169 Get g = new Get(CellUtil.cloneRow(cell));
170 Result r = region.get(g);
171 boolean found = false;
172 for (CellScanner scanner = r.cellScanner(); scanner.advance();) {
173 Cell current = scanner.current();
174 if (CellComparator.compare(cell, current, true) == 0) {
175 found = true;
176 break;
177 }
178 }
179 assertTrue("Failed to find " + cell, found);
180 }
181 }
182 } finally {
183 if (reader != null) reader.close();
184 }
185 return count;
186 }
187 }