View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.util.Map.Entry;
23  import java.util.Properties;
24  import java.util.Set;
25  
26  import org.apache.commons.cli.CommandLine;
27  import org.apache.commons.lang.StringUtils;
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.chaos.factories.MonkeyConstants;
32  import org.apache.hadoop.hbase.chaos.factories.MonkeyFactory;
33  import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
34  import org.apache.hadoop.hbase.util.AbstractHBaseTool;
35  import org.junit.After;
36  import org.junit.Before;
37  
38  /**
39   * Base class for HBase integration tests that want to use the Chaos Monkey.
40   * Usage: bin/hbase <sub_class_of_IntegrationTestBase> <options>
41   * Options: -h,--help Show usage
42   *          -m,--monkey <arg> Which chaos monkey to run
43   *          -monkeyProps <arg> The properties file for specifying chaos monkey properties.
44   *          -ncc Option to not clean up the cluster at the end.
45   */
46  public abstract class IntegrationTestBase extends AbstractHBaseTool {
47  
48    public static final String NO_CLUSTER_CLEANUP_LONG_OPT = "noClusterCleanUp";
49    public static final String MONKEY_LONG_OPT = "monkey";
50    public static final String CHAOS_MONKEY_PROPS = "monkeyProps";
51    private static final Log LOG = LogFactory.getLog(IntegrationTestBase.class);
52  
53    protected IntegrationTestingUtility util;
54    protected ChaosMonkey monkey;
55    protected String monkeyToUse;
56    protected Properties monkeyProps;
57    protected boolean noClusterCleanUp = false;
58  
59    public IntegrationTestBase() {
60      this(null);
61    }
62  
63    public IntegrationTestBase(String monkeyToUse) {
64      this.monkeyToUse = monkeyToUse;
65    }
66  
67    @Override
68    protected void addOptions() {
69      addOptWithArg("m", MONKEY_LONG_OPT, "Which chaos monkey to run");
70      addOptNoArg("ncc", NO_CLUSTER_CLEANUP_LONG_OPT,
71        "Don't clean up the cluster at the end");
72      addOptWithArg(CHAOS_MONKEY_PROPS, "The properties file for specifying chaos "
73          + "monkey properties.");
74    }
75  
76    /**
77     * This allows tests that subclass children of this base class such as
78     * {@link org.apache.hadoop.hbase.test.IntegrationTestReplication} to
79     * include the base options without having to also include the options from the test.
80     *
81     * @param cmd the command line
82     */
83    protected void processBaseOptions(CommandLine cmd) {
84      if (cmd.hasOption(MONKEY_LONG_OPT)) {
85        monkeyToUse = cmd.getOptionValue(MONKEY_LONG_OPT);
86      }
87      if (cmd.hasOption(NO_CLUSTER_CLEANUP_LONG_OPT)) {
88        noClusterCleanUp = true;
89      }
90      monkeyProps = new Properties();
91      // Add entries for the CM from hbase-site.xml as a convenience.
92      // Do this prior to loading from the properties file to make sure those in the properties
93      // file are given precedence to those in hbase-site.xml (backwards compatibility).
94      loadMonkeyProperties(monkeyProps, HBaseConfiguration.create());
95      if (cmd.hasOption(CHAOS_MONKEY_PROPS)) {
96        String chaosMonkeyPropsFile = cmd.getOptionValue(CHAOS_MONKEY_PROPS);
97        if (StringUtils.isNotEmpty(chaosMonkeyPropsFile)) {
98          try {
99            monkeyProps.load(this.getClass().getClassLoader()
100               .getResourceAsStream(chaosMonkeyPropsFile));
101         } catch (IOException e) {
102           LOG.warn(e);
103           System.exit(EXIT_FAILURE);
104         }
105       }
106     }
107   }
108 
109   /**
110    * Loads entries from the provided {@code conf} into {@code props} when the configuration key
111    * is one that may be configuring ChaosMonkey actions.
112    */
113   void loadMonkeyProperties(Properties props, Configuration conf) {
114     for (Entry<String,String> entry : conf) {
115       for (String prefix : MonkeyConstants.MONKEY_CONFIGURATION_KEY_PREFIXES) {
116         if (entry.getKey().startsWith(prefix)) {
117           props.put(entry.getKey(), entry.getValue());
118           break;
119         }
120       }
121     }
122   }
123 
124   @Override
125   protected void processOptions(CommandLine cmd) {
126     processBaseOptions(cmd);
127   }
128 
129   @Override
130   public Configuration getConf() {
131     Configuration c = super.getConf();
132     if (c == null && util != null) {
133       conf = util.getConfiguration();
134       c = conf;
135     }
136     return c;
137   }
138 
139   @Override
140   protected int doWork() throws Exception {
141     setUp();
142     int result = -1;
143     try {
144       result = runTestFromCommandLine();
145     } finally {
146       cleanUp();
147     }
148 
149     return result;
150   }
151 
152   @Before
153   public void setUp() throws Exception {
154     setUpCluster();
155     setUpMonkey();
156   }
157 
158   @After
159   public void cleanUp() throws Exception {
160     cleanUpMonkey();
161     cleanUpCluster();
162   }
163 
164   public void setUpMonkey() throws Exception {
165     util = getTestingUtil(getConf());
166     MonkeyFactory fact = MonkeyFactory.getFactory(monkeyToUse);
167     if (fact == null) {
168       fact = getDefaultMonkeyFactory();
169     }
170     monkey = fact.setUtil(util)
171                  .setTableName(getTablename())
172                  .setProperties(monkeyProps)
173                  .setColumnFamilies(getColumnFamilies()).build();
174     startMonkey();
175   }
176 
177   protected MonkeyFactory getDefaultMonkeyFactory() {
178     // Run with no monkey in distributed context, with real monkey in local test context.
179     return MonkeyFactory.getFactory(
180       util.isDistributedCluster() ? MonkeyFactory.CALM : MonkeyFactory.SLOW_DETERMINISTIC);
181   }
182 
183   protected void startMonkey() throws Exception {
184     monkey.start();
185   }
186 
187   public void cleanUpMonkey() throws Exception {
188     cleanUpMonkey("Ending test");
189   }
190 
191   protected void cleanUpMonkey(String why) throws Exception {
192     if (monkey != null && !monkey.isStopped()) {
193       monkey.stop(why);
194       monkey.waitForStop();
195     }
196   }
197 
198   protected IntegrationTestingUtility getTestingUtil(Configuration conf) {
199     if (this.util == null) {
200       if (conf == null) {
201         this.util = new IntegrationTestingUtility();
202         this.setConf(util.getConfiguration());
203       } else {
204         this.util = new IntegrationTestingUtility(conf);
205       }
206     }
207     return util;
208   }
209 
210   public abstract void setUpCluster() throws Exception;
211 
212   public void cleanUpCluster() throws Exception {
213     if (util.isDistributedCluster() &&  (monkey == null || !monkey.isDestructive())) {
214       noClusterCleanUp = true;
215     }
216     if (noClusterCleanUp) {
217       LOG.debug("noClusterCleanUp is set, skip restoring the cluster");
218       return;
219     }
220     LOG.debug("Restoring the cluster");
221     util.restoreCluster();
222     LOG.debug("Done restoring the cluster");
223   }
224 
225   public abstract int runTestFromCommandLine() throws Exception;
226 
227   public abstract TableName getTablename();
228 
229   protected abstract Set<String> getColumnFamilies();
230 }