1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.procedure;
20
21 import java.io.IOException;
22 import java.util.concurrent.atomic.AtomicInteger;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HRegionInfo;
31 import org.apache.hadoop.hbase.HRegionLocation;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.MetaTableAccessor;
34 import org.apache.hadoop.hbase.RegionLocations;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.TableStateManager;
38 import org.apache.hadoop.hbase.client.BufferedMutator;
39 import org.apache.hadoop.hbase.client.Connection;
40 import org.apache.hadoop.hbase.client.Durability;
41 import org.apache.hadoop.hbase.client.NonceGenerator;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.MetaScanner;
44 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
45 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
46 import org.apache.hadoop.hbase.client.Result;
47 import org.apache.hadoop.hbase.master.HMaster;
48 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
49 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
50 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
51 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.hadoop.hbase.util.MD5Hash;
55
56 import static org.junit.Assert.assertEquals;
57 import static org.junit.Assert.assertFalse;
58 import static org.junit.Assert.assertTrue;
59 import static org.junit.Assert.fail;
60
61 public class MasterProcedureTestingUtility {
62 private static final Log LOG = LogFactory.getLog(MasterProcedureTestingUtility.class);
63
64 private MasterProcedureTestingUtility() {
65 }
66
67 public static HTableDescriptor createHTD(final TableName tableName, final String... family) {
68 HTableDescriptor htd = new HTableDescriptor(tableName);
69 for (int i = 0; i < family.length; ++i) {
70 htd.addFamily(new HColumnDescriptor(family[i]));
71 }
72 return htd;
73 }
74
75 public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv> procExec,
76 final TableName tableName, final byte[][] splitKeys, String... family) throws IOException {
77 HTableDescriptor htd = createHTD(tableName, family);
78 HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
79 long procId = ProcedureTestingUtility.submitAndWait(procExec,
80 new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
81 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
82 return regions;
83 }
84
85 public static void validateTableCreation(final HMaster master, final TableName tableName,
86 final HRegionInfo[] regions, String... family) throws IOException {
87 validateTableCreation(master, tableName, regions, true, family);
88 }
89
90 public static void validateTableCreation(final HMaster master, final TableName tableName,
91 final HRegionInfo[] regions, boolean hasFamilyDirs, String... family) throws IOException {
92
93 final FileSystem fs = master.getMasterFileSystem().getFileSystem();
94 final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
95 assertTrue(fs.exists(tableDir));
96 FSUtils.logFileSystemState(fs, tableDir, LOG);
97 List<Path> allRegionDirs = FSUtils.getRegionDirs(fs, tableDir);
98 for (int i = 0; i < regions.length; ++i) {
99 Path regionDir = new Path(tableDir, regions[i].getEncodedName());
100 if (!fs.exists(regionDir)) {
101 LOG.debug(regions[i] + " region dir does not exist: " + regionDir);
102 }
103 assertTrue(regions[i] + " region dir does not exist: " + regionDir, fs.exists(regionDir));
104 assertTrue(allRegionDirs.remove(regionDir));
105 List<Path> allFamilyDirs = FSUtils.getFamilyDirs(fs, regionDir);
106 for (int j = 0; j < family.length; ++j) {
107 final Path familyDir = new Path(regionDir, family[j]);
108 if (hasFamilyDirs) {
109 assertTrue(family[j] + " family dir does not exist", fs.exists(familyDir));
110 assertTrue(allFamilyDirs.remove(familyDir));
111 } else {
112
113 if (!fs.exists(familyDir)) {
114 LOG.warn(family[j] + " family dir does not exist");
115 }
116 allFamilyDirs.remove(familyDir);
117 }
118 }
119 assertTrue("found extraneous families: " + allFamilyDirs, allFamilyDirs.isEmpty());
120 }
121 assertTrue("found extraneous regions: " + allRegionDirs, allRegionDirs.isEmpty());
122
123
124 assertTrue(MetaTableAccessor.tableExists(master.getConnection(), tableName));
125 assertEquals(regions.length, countMetaRegions(master, tableName));
126
127
128 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
129 assertTrue("table descriptor not found", htd != null);
130 for (int i = 0; i < family.length; ++i) {
131 assertTrue("family not found " + family[i], htd.getFamily(Bytes.toBytes(family[i])) != null);
132 }
133 assertEquals(family.length, htd.getFamilies().size());
134 }
135
136 public static void validateTableDeletion(final HMaster master, final TableName tableName,
137 final HRegionInfo[] regions, String... family) throws IOException {
138
139 final FileSystem fs = master.getMasterFileSystem().getFileSystem();
140 final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
141 assertFalse(fs.exists(tableDir));
142
143
144 assertFalse(MetaTableAccessor.tableExists(master.getConnection(), tableName));
145 assertEquals(0, countMetaRegions(master, tableName));
146
147
148 assertTrue("found htd of deleted table",
149 master.getTableDescriptors().get(tableName) == null);
150 }
151
152 private static int countMetaRegions(final HMaster master, final TableName tableName)
153 throws IOException {
154 final AtomicInteger actualRegCount = new AtomicInteger(0);
155 final MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
156 @Override
157 public boolean processRow(Result rowResult) throws IOException {
158 RegionLocations list = MetaTableAccessor.getRegionLocations(rowResult);
159 if (list == null) {
160 LOG.warn("No serialized HRegionInfo in " + rowResult);
161 return true;
162 }
163 HRegionLocation l = list.getRegionLocation();
164 if (l == null) {
165 return true;
166 }
167 if (!l.getRegionInfo().getTable().equals(tableName)) {
168 return false;
169 }
170 if (l.getRegionInfo().isOffline() || l.getRegionInfo().isSplit()) return true;
171 HRegionLocation[] locations = list.getRegionLocations();
172 for (HRegionLocation location : locations) {
173 if (location == null) continue;
174 ServerName serverName = location.getServerName();
175
176 if (serverName != null && serverName.getHostAndPort() != null) {
177 actualRegCount.incrementAndGet();
178 }
179 }
180 return true;
181 }
182 };
183 MetaScanner.metaScan(master.getConnection(), visitor, tableName);
184 return actualRegCount.get();
185 }
186
187 public static void validateTableIsEnabled(final HMaster master, final TableName tableName)
188 throws IOException {
189 TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
190 assertTrue(tsm.isTableState(tableName, ZooKeeperProtos.Table.State.ENABLED));
191 }
192
193 public static void validateTableIsDisabled(final HMaster master, final TableName tableName)
194 throws IOException {
195 TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
196 assertTrue(tsm.isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED));
197 }
198
199 public static <TState> void testRecoveryAndDoubleExecution(
200 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
201 final int numSteps, final TState[] states) throws Exception {
202 ProcedureTestingUtility.waitProcedure(procExec, procId);
203 assertEquals(false, procExec.isRunning());
204
205
206
207
208 for (int i = 0; i < numSteps; ++i) {
209 LOG.info("Restart "+ i +" exec state: " + states[i]);
210 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
211 ProcedureTestingUtility.restart(procExec);
212 ProcedureTestingUtility.waitProcedure(procExec, procId);
213 }
214 assertEquals(true, procExec.isRunning());
215 ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
216 }
217
218 public static <TState> void testRollbackAndDoubleExecution(
219 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
220 final int lastStep, final TState[] states) throws Exception {
221 ProcedureTestingUtility.waitProcedure(procExec, procId);
222
223
224
225
226
227 for (int i = 0; i < lastStep; ++i) {
228 LOG.info("Restart "+ i +" exec state: " + states[i]);
229 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
230 ProcedureTestingUtility.restart(procExec);
231 ProcedureTestingUtility.waitProcedure(procExec, procId);
232 }
233
234
235
236
237
238 MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
239 new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
240 procExec.registerListener(abortListener);
241 try {
242 for (int i = lastStep + 1; i >= 0; --i) {
243 LOG.info("Restart " + i +" rollback state: "+ states[i]);
244 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
245 ProcedureTestingUtility.restart(procExec);
246 ProcedureTestingUtility.waitProcedure(procExec, procId);
247 }
248 } finally {
249 assertTrue(procExec.unregisterListener(abortListener));
250 }
251
252 ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId));
253 }
254
255 public static <TState> void testRollbackAndDoubleExecutionAfterPONR(
256 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
257 final int lastStep, final TState[] states) throws Exception {
258 ProcedureTestingUtility.waitProcedure(procExec, procId);
259
260
261
262
263
264 for (int i = 0; i < lastStep; ++i) {
265 LOG.info("Restart "+ i +" exec state: " + states[i]);
266 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
267 ProcedureTestingUtility.restart(procExec);
268 ProcedureTestingUtility.waitProcedure(procExec, procId);
269 }
270
271
272 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
273 MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
274 new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
275 procExec.registerListener(abortListener);
276 try {
277 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
278 ProcedureTestingUtility.restart(procExec);
279 LOG.info("Restart and execute");
280 ProcedureTestingUtility.waitProcedure(procExec, procId);
281 } finally {
282 assertTrue(procExec.unregisterListener(abortListener));
283 }
284
285 assertEquals(true, procExec.isRunning());
286 ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
287 }
288
289 public static <TState> void testRollbackRetriableFailure(
290 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
291 final int lastStep, final TState[] states) throws Exception {
292 ProcedureTestingUtility.waitProcedure(procExec, procId);
293
294
295
296
297
298 for (int i = 0; i < lastStep; ++i) {
299 LOG.info("Restart "+ i +" exec state: " + states[i]);
300 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
301 ProcedureTestingUtility.restart(procExec);
302 ProcedureTestingUtility.waitProcedure(procExec, procId);
303 }
304
305
306 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
307 MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
308 new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
309 procExec.registerListener(abortListener);
310 try {
311 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
312 ProcedureTestingUtility.restart(procExec);
313 LOG.info("Restart and rollback");
314 ProcedureTestingUtility.waitProcedure(procExec, procId);
315 } finally {
316 assertTrue(procExec.unregisterListener(abortListener));
317 }
318
319 ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId));
320 }
321
322 public static void validateColumnFamilyAddition(final HMaster master, final TableName tableName,
323 final String family) throws IOException {
324 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
325 assertTrue(htd != null);
326 assertTrue(htd.hasFamily(family.getBytes()));
327 }
328
329 public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName,
330 final String family) throws IOException {
331
332 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
333 assertTrue(htd != null);
334 assertFalse(htd.hasFamily(family.getBytes()));
335
336
337 final FileSystem fs = master.getMasterFileSystem().getFileSystem();
338 final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
339 for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
340 final Path familyDir = new Path(regionDir, family);
341 assertFalse(family + " family dir should not exist", fs.exists(familyDir));
342 }
343 }
344
345 public static void validateColumnFamilyModification(final HMaster master,
346 final TableName tableName, final String family, HColumnDescriptor columnDescriptor)
347 throws IOException {
348 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
349 assertTrue(htd != null);
350
351 HColumnDescriptor hcfd = htd.getFamily(family.getBytes());
352 assertTrue(hcfd.equals(columnDescriptor));
353 }
354
355 public static void loadData(final Connection connection, final TableName tableName,
356 int rows, final byte[][] splitKeys, final String... sfamilies) throws IOException {
357 byte[][] families = new byte[sfamilies.length][];
358 for (int i = 0; i < families.length; ++i) {
359 families[i] = Bytes.toBytes(sfamilies[i]);
360 }
361
362 BufferedMutator mutator = connection.getBufferedMutator(tableName);
363
364
365 assertTrue(rows >= splitKeys.length);
366 for (byte[] k: splitKeys) {
367 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
368 byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
369 mutator.mutate(createPut(families, key, value));
370 rows--;
371 }
372
373
374 while (rows-- > 0) {
375 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
376 byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
377 mutator.mutate(createPut(families, key, value));
378 }
379 mutator.flush();
380 }
381
382 private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
383 byte[] q = Bytes.toBytes("q");
384 Put put = new Put(key);
385 put.setDurability(Durability.SKIP_WAL);
386 for (byte[] family: families) {
387 put.add(family, q, value);
388 }
389 return put;
390 }
391
392 public static long generateNonceGroup(final HMaster master) {
393 return master.getConnection().getNonceGenerator().getNonceGroup();
394 }
395
396 public static long generateNonce(final HMaster master) {
397 return master.getConnection().getNonceGenerator().newNonce();
398 }
399
400 public static class InjectAbortOnLoadListener
401 implements ProcedureExecutor.ProcedureExecutorListener {
402 private final ProcedureExecutor<MasterProcedureEnv> procExec;
403
404 public InjectAbortOnLoadListener(final ProcedureExecutor<MasterProcedureEnv> procExec) {
405 this.procExec = procExec;
406 }
407
408 @Override
409 public void procedureLoaded(long procId) {
410 procExec.abort(procId);
411 }
412
413 @Override
414 public void procedureAdded(long procId) {
415
416 @Override
417 public void procedureFinished(long procId) {
418 }
419 }