View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.security.token;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.util.concurrent.CountDownLatch;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.Abortable;
33  import org.apache.hadoop.hbase.HBaseConfiguration;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.testclassification.LargeTests;
36  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
37  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38  import org.junit.AfterClass;
39  import org.junit.BeforeClass;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  /**
44   * Test the synchronization of token authentication master keys through
45   * ZKSecretWatcher
46   */
47  @Category(LargeTests.class)
48  public class TestZKSecretWatcher {
49    private static Log LOG = LogFactory.getLog(TestZKSecretWatcher.class);
50    private static HBaseTestingUtility TEST_UTIL;
51    private static AuthenticationTokenSecretManager KEY_MASTER;
52    private static AuthenticationTokenSecretManagerForTest KEY_SLAVE;
53    private static AuthenticationTokenSecretManager KEY_SLAVE2;
54    private static AuthenticationTokenSecretManager KEY_SLAVE3;
55  
56    private static class MockAbortable implements Abortable {
57      private boolean abort;
58      public void abort(String reason, Throwable e) {
59        LOG.info("Aborting: "+reason, e);
60        abort = true;
61      }
62  
63      public boolean isAborted() {
64        return abort;
65      }
66    }
67  
68    // We subclass AuthenticationTokenSecretManager so that testKeyUpdate can receive
69    // notification on the removal of keyId
70    private static class AuthenticationTokenSecretManagerForTest
71    extends AuthenticationTokenSecretManager {
72      private CountDownLatch latch = new CountDownLatch(1);
73      
74      public AuthenticationTokenSecretManagerForTest(Configuration conf,
75          ZooKeeperWatcher zk, String serverName,
76          long keyUpdateInterval, long tokenMaxLifetime) {
77        super(conf, zk, serverName, keyUpdateInterval, tokenMaxLifetime);
78      }
79      
80      @Override
81      synchronized boolean removeKey(Integer keyId) {
82        boolean b = super.removeKey(keyId);
83        if (b) {
84          latch.countDown();
85        }
86        return b;
87      }
88      
89      CountDownLatch getLatch() {
90        return latch;
91      }
92    }
93    
94    @BeforeClass
95    public static void setupBeforeClass() throws Exception {
96      TEST_UTIL = new HBaseTestingUtility();
97      TEST_UTIL.startMiniZKCluster();
98      Configuration conf = TEST_UTIL.getConfiguration();
99  
100     ZooKeeperWatcher zk = newZK(conf, "server1", new MockAbortable());
101     AuthenticationTokenSecretManagerForTest[] tmp = new AuthenticationTokenSecretManagerForTest[2];
102     tmp[0] = new AuthenticationTokenSecretManagerForTest(
103         conf, zk, "server1", 60*60*1000, 60*1000);
104     tmp[0].start();
105 
106     zk = newZK(conf, "server2", new MockAbortable());
107     tmp[1] = new AuthenticationTokenSecretManagerForTest(
108         conf, zk, "server2", 60*60*1000, 60*1000);
109     tmp[1].start();
110 
111     while (KEY_MASTER == null) {
112       for (int i=0; i<2; i++) {
113         if (tmp[i].isMaster()) {
114           KEY_MASTER = tmp[i];
115           KEY_SLAVE = tmp[ (i+1) % 2 ];
116           break;
117         }
118       }
119       Thread.sleep(500);
120     }
121     LOG.info("Master is "+KEY_MASTER.getName()+
122         ", slave is "+KEY_SLAVE.getName());
123   }
124 
125   @AfterClass
126   public static void tearDownAfterClass() throws Exception {
127     TEST_UTIL.shutdownMiniZKCluster();
128   }
129 
130   @Test
131   public void testKeyUpdate() throws Exception {
132     // sanity check
133     assertTrue(KEY_MASTER.isMaster());
134     assertFalse(KEY_SLAVE.isMaster());
135     int maxKeyId = 0;
136 
137     KEY_MASTER.rollCurrentKey();
138     AuthenticationKey key1 = KEY_MASTER.getCurrentKey();
139     assertNotNull(key1);
140     LOG.debug("Master current key: "+key1.getKeyId());
141 
142     // wait for slave to update
143     Thread.sleep(1000);
144     AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey();
145     assertNotNull(slaveCurrent);
146     assertEquals(key1, slaveCurrent);
147     LOG.debug("Slave current key: "+slaveCurrent.getKeyId());
148 
149     // generate two more keys then expire the original
150     KEY_MASTER.rollCurrentKey();
151     AuthenticationKey key2 = KEY_MASTER.getCurrentKey();
152     LOG.debug("Master new current key: "+key2.getKeyId());
153     KEY_MASTER.rollCurrentKey();
154     AuthenticationKey key3 = KEY_MASTER.getCurrentKey();
155     LOG.debug("Master new current key: "+key3.getKeyId());
156 
157     // force expire the original key
158     key1.setExpiration(EnvironmentEdgeManager.currentTime() - 1000);
159     KEY_MASTER.removeExpiredKeys();
160     // verify removed from master
161     assertNull(KEY_MASTER.getKey(key1.getKeyId()));
162 
163     // wait for slave to catch up
164     KEY_SLAVE.getLatch().await();
165     // make sure the slave has both new keys
166     AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId());
167     assertNotNull(slave2);
168     assertEquals(key2, slave2);
169     AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId());
170     assertNotNull(slave3);
171     assertEquals(key3, slave3);
172     slaveCurrent = KEY_SLAVE.getCurrentKey();
173     assertEquals(key3, slaveCurrent);
174     LOG.debug("Slave current key: "+slaveCurrent.getKeyId());
175 
176     // verify that the expired key has been removed
177     assertNull(KEY_SLAVE.getKey(key1.getKeyId()));
178 
179     // bring up a new slave
180     Configuration conf = TEST_UTIL.getConfiguration();
181     ZooKeeperWatcher zk = newZK(conf, "server3", new MockAbortable());
182     KEY_SLAVE2 = new AuthenticationTokenSecretManager(
183         conf, zk, "server3", 60*60*1000, 60*1000);
184     KEY_SLAVE2.start();
185 
186     Thread.sleep(1000);
187     // verify the new slave has current keys (and not expired)
188     slave2 = KEY_SLAVE2.getKey(key2.getKeyId());
189     assertNotNull(slave2);
190     assertEquals(key2, slave2);
191     slave3 = KEY_SLAVE2.getKey(key3.getKeyId());
192     assertNotNull(slave3);
193     assertEquals(key3, slave3);
194     slaveCurrent = KEY_SLAVE2.getCurrentKey();
195     assertEquals(key3, slaveCurrent);
196     assertNull(KEY_SLAVE2.getKey(key1.getKeyId()));
197 
198     // test leader failover
199     KEY_MASTER.stop();
200 
201     // wait for master to stop
202     Thread.sleep(1000);
203     assertFalse(KEY_MASTER.isMaster());
204 
205     // check for a new master
206     AuthenticationTokenSecretManager[] mgrs =
207         new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2 };
208     AuthenticationTokenSecretManager newMaster = null;
209     int tries = 0;
210     while (newMaster == null && tries++ < 5) {
211       for (AuthenticationTokenSecretManager mgr : mgrs) {
212         if (mgr.isMaster()) {
213           newMaster = mgr;
214           break;
215         }
216       }
217       if (newMaster == null) {
218         Thread.sleep(500);
219       }
220     }
221     assertNotNull(newMaster);
222 
223     AuthenticationKey current = newMaster.getCurrentKey();
224     // new master will immediately roll the current key, so it's current may be greater
225     assertTrue(current.getKeyId() >= slaveCurrent.getKeyId());
226     LOG.debug("New master, current key: "+current.getKeyId());
227 
228     // roll the current key again on new master and verify the key ID increments
229     newMaster.rollCurrentKey();
230     AuthenticationKey newCurrent = newMaster.getCurrentKey();
231     LOG.debug("New master, rolled new current key: "+newCurrent.getKeyId());
232     assertTrue(newCurrent.getKeyId() > current.getKeyId());
233 
234     // add another slave
235     ZooKeeperWatcher zk3 = newZK(conf, "server4", new MockAbortable());
236     KEY_SLAVE3 = new AuthenticationTokenSecretManager(
237         conf, zk3, "server4", 60*60*1000, 60*1000);
238     KEY_SLAVE3.start();
239     Thread.sleep(5000);
240 
241     // check master failover again
242     newMaster.stop();
243 
244     // wait for master to stop
245     Thread.sleep(5000);
246     assertFalse(newMaster.isMaster());
247 
248     // check for a new master
249     mgrs = new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 };
250     newMaster = null;
251     tries = 0;
252     while (newMaster == null && tries++ < 5) {
253       for (AuthenticationTokenSecretManager mgr : mgrs) {
254         if (mgr.isMaster()) {
255           newMaster = mgr;
256           break;
257         }
258       }
259       if (newMaster == null) {
260         Thread.sleep(500);
261       }
262     }
263     assertNotNull(newMaster);
264 
265     AuthenticationKey current2 = newMaster.getCurrentKey();
266     // new master will immediately roll the current key, so it's current may be greater
267     assertTrue(current2.getKeyId() >= newCurrent.getKeyId());
268     LOG.debug("New master 2, current key: "+current2.getKeyId());
269 
270     // roll the current key again on new master and verify the key ID increments
271     newMaster.rollCurrentKey();
272     AuthenticationKey newCurrent2 = newMaster.getCurrentKey();
273     LOG.debug("New master 2, rolled new current key: "+newCurrent2.getKeyId());
274     assertTrue(newCurrent2.getKeyId() > current2.getKeyId());
275   }
276 
277   private static ZooKeeperWatcher newZK(Configuration conf, String name,
278       Abortable abort) throws Exception {
279     Configuration copy = HBaseConfiguration.create(conf);
280     ZooKeeperWatcher zk = new ZooKeeperWatcher(copy, name, abort);
281     return zk;
282   }
283 }