View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import java.io.IOException;
22  import java.util.concurrent.atomic.AtomicInteger;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HRegionLocation;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.MetaTableAccessor;
34  import org.apache.hadoop.hbase.RegionLocations;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.TableStateManager;
38  import org.apache.hadoop.hbase.client.BufferedMutator;
39  import org.apache.hadoop.hbase.client.Connection;
40  import org.apache.hadoop.hbase.client.Durability;
41  import org.apache.hadoop.hbase.client.NonceGenerator;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.MetaScanner;
44  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
45  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
46  import org.apache.hadoop.hbase.client.Result;
47  import org.apache.hadoop.hbase.master.HMaster;
48  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
49  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
50  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
51  import org.apache.hadoop.hbase.util.ModifyRegionUtils;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  import org.apache.hadoop.hbase.util.MD5Hash;
55  
56  import static org.junit.Assert.assertEquals;
57  import static org.junit.Assert.assertFalse;
58  import static org.junit.Assert.assertTrue;
59  import static org.junit.Assert.fail;
60  
61  public class MasterProcedureTestingUtility {
62    private static final Log LOG = LogFactory.getLog(MasterProcedureTestingUtility.class);
63  
64    private MasterProcedureTestingUtility() {
65    }
66  
67    public static HTableDescriptor createHTD(final TableName tableName, final String... family) {
68      HTableDescriptor htd = new HTableDescriptor(tableName);
69      for (int i = 0; i < family.length; ++i) {
70        htd.addFamily(new HColumnDescriptor(family[i]));
71      }
72      return htd;
73    }
74  
75    public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv> procExec,
76        final TableName tableName, final byte[][] splitKeys, String... family) throws IOException {
77      HTableDescriptor htd = createHTD(tableName, family);
78      HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
79      long procId = ProcedureTestingUtility.submitAndWait(procExec,
80        new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
81      ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
82      return regions;
83    }
84  
85    public static void validateTableCreation(final HMaster master, final TableName tableName,
86        final HRegionInfo[] regions, String... family) throws IOException {
87      validateTableCreation(master, tableName, regions, true, family);
88    }
89  
90    public static void validateTableCreation(final HMaster master, final TableName tableName,
91        final HRegionInfo[] regions, boolean hasFamilyDirs, String... family) throws IOException {
92      // check filesystem
93      final FileSystem fs = master.getMasterFileSystem().getFileSystem();
94      final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
95      assertTrue(fs.exists(tableDir));
96      FSUtils.logFileSystemState(fs, tableDir, LOG);
97      List<Path> allRegionDirs = FSUtils.getRegionDirs(fs, tableDir);
98      for (int i = 0; i < regions.length; ++i) {
99        Path regionDir = new Path(tableDir, regions[i].getEncodedName());
100       if (!fs.exists(regionDir)) {
101         LOG.debug(regions[i] + " region dir does not exist: " + regionDir);
102       }
103       assertTrue(regions[i] + " region dir does not exist: " + regionDir, fs.exists(regionDir));
104       assertTrue(allRegionDirs.remove(regionDir));
105       List<Path> allFamilyDirs = FSUtils.getFamilyDirs(fs, regionDir);
106       for (int j = 0; j < family.length; ++j) {
107         final Path familyDir = new Path(regionDir, family[j]);
108         if (hasFamilyDirs) {
109           assertTrue(family[j] + " family dir does not exist", fs.exists(familyDir));
110           assertTrue(allFamilyDirs.remove(familyDir));
111         } else {
112           // TODO: WARN: Modify Table/Families does not create a family dir
113           if (!fs.exists(familyDir)) {
114             LOG.warn(family[j] + " family dir does not exist");
115           }
116           allFamilyDirs.remove(familyDir);
117         }
118       }
119       assertTrue("found extraneous families: " + allFamilyDirs, allFamilyDirs.isEmpty());
120     }
121     assertTrue("found extraneous regions: " + allRegionDirs, allRegionDirs.isEmpty());
122 
123     // check meta
124     assertTrue(MetaTableAccessor.tableExists(master.getConnection(), tableName));
125     assertEquals(regions.length, countMetaRegions(master, tableName));
126 
127     // check htd
128     HTableDescriptor htd = master.getTableDescriptors().get(tableName);
129     assertTrue("table descriptor not found", htd != null);
130     for (int i = 0; i < family.length; ++i) {
131       assertTrue("family not found " + family[i], htd.getFamily(Bytes.toBytes(family[i])) != null);
132     }
133     assertEquals(family.length, htd.getFamilies().size());
134   }
135 
136   public static void validateTableDeletion(final HMaster master, final TableName tableName,
137       final HRegionInfo[] regions, String... family) throws IOException {
138     // check filesystem
139     final FileSystem fs = master.getMasterFileSystem().getFileSystem();
140     final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
141     assertFalse(fs.exists(tableDir));
142 
143     // check meta
144     assertFalse(MetaTableAccessor.tableExists(master.getConnection(), tableName));
145     assertEquals(0, countMetaRegions(master, tableName));
146 
147     // check htd
148     assertTrue("found htd of deleted table",
149       master.getTableDescriptors().get(tableName) == null);
150   }
151 
152   private static int countMetaRegions(final HMaster master, final TableName tableName)
153       throws IOException {
154     final AtomicInteger actualRegCount = new AtomicInteger(0);
155     final MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
156       @Override
157       public boolean processRow(Result rowResult) throws IOException {
158         RegionLocations list = MetaTableAccessor.getRegionLocations(rowResult);
159         if (list == null) {
160           LOG.warn("No serialized HRegionInfo in " + rowResult);
161           return true;
162         }
163         HRegionLocation l = list.getRegionLocation();
164         if (l == null) {
165           return true;
166         }
167         if (!l.getRegionInfo().getTable().equals(tableName)) {
168           return false;
169         }
170         if (l.getRegionInfo().isOffline() || l.getRegionInfo().isSplit()) return true;
171         HRegionLocation[] locations = list.getRegionLocations();
172         for (HRegionLocation location : locations) {
173           if (location == null) continue;
174           ServerName serverName = location.getServerName();
175           // Make sure that regions are assigned to server
176           if (serverName != null && serverName.getHostAndPort() != null) {
177             actualRegCount.incrementAndGet();
178           }
179         }
180         return true;
181       }
182     };
183     MetaScanner.metaScan(master.getConnection(), visitor, tableName);
184     return actualRegCount.get();
185   }
186 
187   public static void validateTableIsEnabled(final HMaster master, final TableName tableName)
188       throws IOException {
189     TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
190     assertTrue(tsm.isTableState(tableName, ZooKeeperProtos.Table.State.ENABLED));
191   }
192 
193   public static void validateTableIsDisabled(final HMaster master, final TableName tableName)
194       throws IOException {
195     TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
196     assertTrue(tsm.isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED));
197   }
198 
199   public static <TState> void testRecoveryAndDoubleExecution(
200       final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
201       final int numSteps, final TState[] states) throws Exception {
202     ProcedureTestingUtility.waitProcedure(procExec, procId);
203     assertEquals(false, procExec.isRunning());
204     // Restart the executor and execute the step twice
205     //   execute step N - kill before store update
206     //   restart executor/store
207     //   execute step N - save on store
208     for (int i = 0; i < numSteps; ++i) {
209       LOG.info("Restart "+ i +" exec state: " + states[i]);
210       ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
211       ProcedureTestingUtility.restart(procExec);
212       ProcedureTestingUtility.waitProcedure(procExec, procId);
213     }
214     assertEquals(true, procExec.isRunning());
215     ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
216   }
217 
218   public static <TState> void testRollbackAndDoubleExecution(
219       final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
220       final int lastStep, final TState[] states) throws Exception {
221     ProcedureTestingUtility.waitProcedure(procExec, procId);
222 
223     // Restart the executor and execute the step twice
224     //   execute step N - kill before store update
225     //   restart executor/store
226     //   execute step N - save on store
227     for (int i = 0; i < lastStep; ++i) {
228       LOG.info("Restart "+ i +" exec state: " + states[i]);
229       ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
230       ProcedureTestingUtility.restart(procExec);
231       ProcedureTestingUtility.waitProcedure(procExec, procId);
232     }
233 
234     // Restart the executor and rollback the step twice
235     //   rollback step N - kill before store update
236     //   restart executor/store
237     //   rollback step N - save on store
238     MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
239       new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
240     procExec.registerListener(abortListener);
241     try {
242       for (int i = lastStep + 1; i >= 0; --i) {
243         LOG.info("Restart " + i +" rollback state: "+ states[i]);
244         ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
245         ProcedureTestingUtility.restart(procExec);
246         ProcedureTestingUtility.waitProcedure(procExec, procId);
247       }
248     } finally {
249       assertTrue(procExec.unregisterListener(abortListener));
250     }
251 
252     ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId));
253   }
254 
255   public static <TState> void testRollbackAndDoubleExecutionAfterPONR(
256       final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
257       final int lastStep, final TState[] states) throws Exception {
258     ProcedureTestingUtility.waitProcedure(procExec, procId);
259 
260     // Restart the executor and execute the step twice
261     //   execute step N - kill before store update
262     //   restart executor/store
263     //   execute step N - save on store
264     for (int i = 0; i < lastStep; ++i) {
265       LOG.info("Restart "+ i +" exec state: " + states[i]);
266       ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
267       ProcedureTestingUtility.restart(procExec);
268       ProcedureTestingUtility.waitProcedure(procExec, procId);
269     }
270 
271     // try to inject the abort
272     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
273     MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
274       new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
275     procExec.registerListener(abortListener);
276     try {
277       ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
278       ProcedureTestingUtility.restart(procExec);
279       LOG.info("Restart and execute");
280       ProcedureTestingUtility.waitProcedure(procExec, procId);
281     } finally {
282       assertTrue(procExec.unregisterListener(abortListener));
283     }
284 
285     assertEquals(true, procExec.isRunning());
286     ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
287   }
288 
289   public static <TState> void testRollbackRetriableFailure(
290       final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
291       final int lastStep, final TState[] states) throws Exception {
292     ProcedureTestingUtility.waitProcedure(procExec, procId);
293 
294     // Restart the executor and execute the step twice
295     //   execute step N - kill before store update
296     //   restart executor/store
297     //   execute step N - save on store
298     for (int i = 0; i < lastStep; ++i) {
299       LOG.info("Restart "+ i +" exec state: " + states[i]);
300       ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
301       ProcedureTestingUtility.restart(procExec);
302       ProcedureTestingUtility.waitProcedure(procExec, procId);
303     }
304 
305     // execute the rollback
306     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
307     MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
308       new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
309     procExec.registerListener(abortListener);
310     try {
311       ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
312       ProcedureTestingUtility.restart(procExec);
313       LOG.info("Restart and rollback");
314       ProcedureTestingUtility.waitProcedure(procExec, procId);
315     } finally {
316       assertTrue(procExec.unregisterListener(abortListener));
317     }
318 
319     ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId));
320   }
321 
322   public static void validateColumnFamilyAddition(final HMaster master, final TableName tableName,
323       final String family) throws IOException {
324     HTableDescriptor htd = master.getTableDescriptors().get(tableName);
325     assertTrue(htd != null);
326     assertTrue(htd.hasFamily(family.getBytes()));
327   }
328 
329   public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName,
330       final String family) throws IOException {
331     // verify htd
332     HTableDescriptor htd = master.getTableDescriptors().get(tableName);
333     assertTrue(htd != null);
334     assertFalse(htd.hasFamily(family.getBytes()));
335 
336     // verify fs
337     final FileSystem fs = master.getMasterFileSystem().getFileSystem();
338     final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
339     for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
340       final Path familyDir = new Path(regionDir, family);
341       assertFalse(family + " family dir should not exist", fs.exists(familyDir));
342     }
343   }
344 
345   public static void validateColumnFamilyModification(final HMaster master,
346       final TableName tableName, final String family, HColumnDescriptor columnDescriptor)
347       throws IOException {
348     HTableDescriptor htd = master.getTableDescriptors().get(tableName);
349     assertTrue(htd != null);
350 
351     HColumnDescriptor hcfd = htd.getFamily(family.getBytes());
352     assertTrue(hcfd.equals(columnDescriptor));
353   }
354 
355   public static void loadData(final Connection connection, final TableName tableName,
356       int rows, final byte[][] splitKeys,  final String... sfamilies) throws IOException {
357     byte[][] families = new byte[sfamilies.length][];
358     for (int i = 0; i < families.length; ++i) {
359       families[i] = Bytes.toBytes(sfamilies[i]);
360     }
361 
362     BufferedMutator mutator = connection.getBufferedMutator(tableName);
363 
364     // Ensure one row per region
365     assertTrue(rows >= splitKeys.length);
366     for (byte[] k: splitKeys) {
367       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
368       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
369       mutator.mutate(createPut(families, key, value));
370       rows--;
371     }
372 
373     // Add other extra rows. more rows, more files
374     while (rows-- > 0) {
375       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
376       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
377       mutator.mutate(createPut(families, key, value));
378     }
379     mutator.flush();
380   }
381 
382   private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
383     byte[] q = Bytes.toBytes("q");
384     Put put = new Put(key);
385     put.setDurability(Durability.SKIP_WAL);
386     for (byte[] family: families) {
387       put.add(family, q, value);
388     }
389     return put;
390   }
391 
392   public static long generateNonceGroup(final HMaster master) {
393     return master.getConnection().getNonceGenerator().getNonceGroup();
394   }
395 
396   public static long generateNonce(final HMaster master) {
397     return master.getConnection().getNonceGenerator().newNonce();
398   }
399 
400   public static class InjectAbortOnLoadListener
401       implements ProcedureExecutor.ProcedureExecutorListener {
402     private final ProcedureExecutor<MasterProcedureEnv> procExec;
403 
404     public InjectAbortOnLoadListener(final ProcedureExecutor<MasterProcedureEnv> procExec) {
405       this.procExec = procExec;
406     }
407 
408     @Override
409     public void procedureLoaded(long procId) {
410       procExec.abort(procId);
411     }
412 
413     @Override
414     public void procedureAdded(long procId) { /* no-op */ }
415 
416     @Override
417     public void procedureFinished(long procId) { /* no-op */ }
418   }
419 }