View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.BufferedReader;
22  import java.io.IOException;
23  import java.io.InputStreamReader;
24  import java.io.PrintWriter;
25  import java.net.InetSocketAddress;
26  import java.net.Socket;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Deque;
30  import java.util.HashMap;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Map;
34  
35  import javax.security.auth.login.AppConfigurationEntry;
36  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
37  
38  import org.apache.commons.lang.StringUtils;
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.AuthUtil;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.classification.InterfaceAudience;
47  import org.apache.hadoop.hbase.exceptions.DeserializationException;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
50  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
51  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
52  import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
53  import org.apache.hadoop.hbase.security.Superusers;
54  import org.apache.hadoop.hbase.util.ByteStringer;
55  import org.apache.hadoop.hbase.util.Bytes;
56  import org.apache.hadoop.hbase.util.Threads;
57  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
58  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
59  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
60  import org.apache.hadoop.security.SecurityUtil;
61  import org.apache.hadoop.security.authentication.util.KerberosUtil;
62  import org.apache.zookeeper.AsyncCallback;
63  import org.apache.zookeeper.CreateMode;
64  import org.apache.zookeeper.KeeperException;
65  import org.apache.zookeeper.KeeperException.NoNodeException;
66  import org.apache.zookeeper.Op;
67  import org.apache.zookeeper.Watcher;
68  import org.apache.zookeeper.ZooDefs.Ids;
69  import org.apache.zookeeper.ZooDefs.Perms;
70  import org.apache.zookeeper.ZooKeeper;
71  import org.apache.zookeeper.client.ZooKeeperSaslClient;
72  import org.apache.zookeeper.data.ACL;
73  import org.apache.zookeeper.data.Id;
74  import org.apache.zookeeper.data.Stat;
75  import org.apache.zookeeper.proto.CreateRequest;
76  import org.apache.zookeeper.proto.DeleteRequest;
77  import org.apache.zookeeper.proto.SetDataRequest;
78  import org.apache.zookeeper.server.ZooKeeperSaslServer;
79  
80  import com.google.protobuf.InvalidProtocolBufferException;
81  
82  /**
83   * Internal HBase utility class for ZooKeeper.
84   *
85   * <p>Contains only static methods and constants.
86   *
87   * <p>Methods all throw {@link KeeperException} if there is an unexpected
88   * zookeeper exception, so callers of these methods must handle appropriately.
89   * If ZK is required for the operation, the server will need to be aborted.
90   */
91  @InterfaceAudience.Private
92  public class ZKUtil {
93    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
94  
95    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
96    public static final char ZNODE_PATH_SEPARATOR = '/';
97    private static int zkDumpConnectionTimeOut;
98  
99    /**
100    * Creates a new connection to ZooKeeper, pulling settings and ensemble config
101    * from the specified configuration object using methods from {@link ZKConfig}.
102    *
103    * Sets the connection status monitoring watcher to the specified watcher.
104    *
105    * @param conf configuration to pull ensemble and other settings from
106    * @param watcher watcher to monitor connection changes
107    * @return connection to zookeeper
108    * @throws IOException if unable to connect to zk or config problem
109    */
110   public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
111   throws IOException {
112     String ensemble = ZKConfig.getZKQuorumServersString(conf);
113     return connect(conf, ensemble, watcher);
114   }
115 
116   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
117       Watcher watcher)
118   throws IOException {
119     return connect(conf, ensemble, watcher, null);
120   }
121 
122   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
123       Watcher watcher, final String identifier)
124   throws IOException {
125     if(ensemble == null) {
126       throw new IOException("Unable to determine ZooKeeper ensemble");
127     }
128     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
129         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
130     if (LOG.isTraceEnabled()) {
131       LOG.trace(identifier + " opening connection to ZooKeeper ensemble=" + ensemble);
132     }
133     int retry = conf.getInt("zookeeper.recovery.retry", 3);
134     int retryIntervalMillis =
135       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
136     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
137         1000);
138     return new RecoverableZooKeeper(ensemble, timeout, watcher,
139         retry, retryIntervalMillis, identifier);
140   }
141 
142   /**
143    * Log in the current zookeeper server process using the given configuration
144    * keys for the credential file and login principal.
145    *
146    * <p><strong>This is only applicable when running on secure hbase</strong>
147    * On regular HBase (without security features), this will safely be ignored.
148    * </p>
149    *
150    * @param conf The configuration data to use
151    * @param keytabFileKey Property key used to configure the path to the credential file
152    * @param userNameKey Property key used to configure the login principal
153    * @param hostname Current hostname to use in any credentials
154    * @throws IOException underlying exception from SecurityUtil.login() call
155    */
156   public static void loginServer(Configuration conf, String keytabFileKey,
157       String userNameKey, String hostname) throws IOException {
158     login(conf, keytabFileKey, userNameKey, hostname,
159           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
160           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
161   }
162 
163   /**
164    * Log in the current zookeeper client using the given configuration
165    * keys for the credential file and login principal.
166    *
167    * <p><strong>This is only applicable when running on secure hbase</strong>
168    * On regular HBase (without security features), this will safely be ignored.
169    * </p>
170    *
171    * @param conf The configuration data to use
172    * @param keytabFileKey Property key used to configure the path to the credential file
173    * @param userNameKey Property key used to configure the login principal
174    * @param hostname Current hostname to use in any credentials
175    * @throws IOException underlying exception from SecurityUtil.login() call
176    */
177   public static void loginClient(Configuration conf, String keytabFileKey,
178       String userNameKey, String hostname) throws IOException {
179     login(conf, keytabFileKey, userNameKey, hostname,
180           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
181           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
182   }
183 
184   /**
185    * Log in the current process using the given configuration keys for the
186    * credential file and login principal.
187    *
188    * <p><strong>This is only applicable when running on secure hbase</strong>
189    * On regular HBase (without security features), this will safely be ignored.
190    * </p>
191    *
192    * @param conf The configuration data to use
193    * @param keytabFileKey Property key used to configure the path to the credential file
194    * @param userNameKey Property key used to configure the login principal
195    * @param hostname Current hostname to use in any credentials
196    * @param loginContextProperty property name to expose the entry name
197    * @param loginContextName jaas entry name
198    * @throws IOException underlying exception from SecurityUtil.login() call
199    */
200   private static void login(Configuration conf, String keytabFileKey,
201       String userNameKey, String hostname,
202       String loginContextProperty, String loginContextName)
203       throws IOException {
204     if (!isSecureZooKeeper(conf))
205       return;
206 
207     // User has specified a jaas.conf, keep this one as the good one.
208     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
209     if (System.getProperty("java.security.auth.login.config") != null)
210       return;
211 
212     // No keytab specified, no auth
213     String keytabFilename = conf.get(keytabFileKey);
214     if (keytabFilename == null) {
215       LOG.warn("no keytab specified for: " + keytabFileKey);
216       return;
217     }
218 
219     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
220     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
221 
222     // Initialize the "jaas.conf" for keyTab/principal,
223     // If keyTab is not specified use the Ticket Cache.
224     // and set the zookeeper login context name.
225     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
226         principalName, keytabFilename);
227     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
228     System.setProperty(loginContextProperty, loginContextName);
229   }
230 
231   /**
232    * A JAAS configuration that defines the login modules that we want to use for login.
233    */
234   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
235     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
236       "zookeeper-server-keytab-kerberos";
237     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
238       "zookeeper-client-keytab-kerberos";
239 
240     private static final Map<String, String> BASIC_JAAS_OPTIONS =
241       new HashMap<String,String>();
242     static {
243       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
244       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
245         BASIC_JAAS_OPTIONS.put("debug", "true");
246       }
247     }
248 
249     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
250       new HashMap<String,String>();
251     static {
252       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
253       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
254       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
255       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
256     }
257 
258     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
259       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
260                                 LoginModuleControlFlag.REQUIRED,
261                                 KEYTAB_KERBEROS_OPTIONS);
262 
263     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
264       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
265 
266     private javax.security.auth.login.Configuration baseConfig;
267     private final String loginContextName;
268     private final boolean useTicketCache;
269     private final String keytabFile;
270     private final String principal;
271 
272     public JaasConfiguration(String loginContextName, String principal) {
273       this(loginContextName, principal, null, true);
274     }
275 
276     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
277       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
278     }
279 
280     private JaasConfiguration(String loginContextName, String principal,
281                              String keytabFile, boolean useTicketCache) {
282       try {
283         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
284       } catch (SecurityException e) {
285         this.baseConfig = null;
286       }
287       this.loginContextName = loginContextName;
288       this.useTicketCache = useTicketCache;
289       this.keytabFile = keytabFile;
290       this.principal = principal;
291       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
292                " principal=" + principal + " useTicketCache=" + useTicketCache +
293                " keytabFile=" + keytabFile);
294     }
295 
296     @Override
297     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
298       if (loginContextName.equals(appName)) {
299         if (!useTicketCache) {
300           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
301           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
302         }
303         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
304         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
305         return KEYTAB_KERBEROS_CONF;
306       }
307       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
308       return(null);
309     }
310   }
311 
312   //
313   // Helper methods
314   //
315 
316   /**
317    * Join the prefix znode name with the suffix znode name to generate a proper
318    * full znode name.
319    *
320    * Assumes prefix does not end with slash and suffix does not begin with it.
321    *
322    * @param prefix beginning of znode name
323    * @param suffix ending of znode name
324    * @return result of properly joining prefix with suffix
325    */
326   public static String joinZNode(String prefix, String suffix) {
327     return prefix + ZNODE_PATH_SEPARATOR + suffix;
328   }
329 
330   /**
331    * Returns the full path of the immediate parent of the specified node.
332    * @param node path to get parent of
333    * @return parent of path, null if passed the root node or an invalid node
334    */
335   public static String getParent(String node) {
336     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
337     return idx <= 0 ? null : node.substring(0, idx);
338   }
339 
340   /**
341    * Get the name of the current node from the specified fully-qualified path.
342    * @param path fully-qualified path
343    * @return name of the current node
344    */
345   public static String getNodeName(String path) {
346     return path.substring(path.lastIndexOf("/")+1);
347   }
348 
349   //
350   // Existence checks and watches
351   //
352 
353   /**
354    * Watch the specified znode for delete/create/change events.  The watcher is
355    * set whether or not the node exists.  If the node already exists, the method
356    * returns true.  If the node does not exist, the method returns false.
357    *
358    * @param zkw zk reference
359    * @param znode path of node to watch
360    * @return true if znode exists, false if does not exist or error
361    * @throws KeeperException if unexpected zookeeper exception
362    */
363   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
364   throws KeeperException {
365     try {
366       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
367       boolean exists = s != null ? true : false;
368       if (exists) {
369         LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
370       } else {
371         LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
372       }
373       return exists;
374     } catch (KeeperException e) {
375       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
376       zkw.keeperException(e);
377       return false;
378     } catch (InterruptedException e) {
379       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
380       zkw.interruptedException(e);
381       return false;
382     }
383   }
384 
385   /**
386    * Watch the specified znode, but only if exists. Useful when watching
387    * for deletions. Uses .getData() (and handles NoNodeException) instead
388    * of .exists() to accomplish this, as .getData() will only set a watch if
389    * the znode exists.
390    * @param zkw zk reference
391    * @param znode path of node to watch
392    * @return true if the watch is set, false if node does not exists
393    * @throws KeeperException if unexpected zookeeper exception
394    */
395   public static boolean setWatchIfNodeExists(ZooKeeperWatcher zkw, String znode)
396       throws KeeperException {
397     try {
398       zkw.getRecoverableZooKeeper().getData(znode, true, null);
399       return true;
400     } catch (NoNodeException e) {
401       return false;
402     } catch (InterruptedException e) {
403       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
404       zkw.interruptedException(e);
405       return false;
406     }
407   }
408 
409   /**
410    * Check if the specified node exists.  Sets no watches.
411    *
412    * @param zkw zk reference
413    * @param znode path of node to watch
414    * @return version of the node if it exists, -1 if does not exist
415    * @throws KeeperException if unexpected zookeeper exception
416    */
417   public static int checkExists(ZooKeeperWatcher zkw, String znode)
418   throws KeeperException {
419     try {
420       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
421       return s != null ? s.getVersion() : -1;
422     } catch (KeeperException e) {
423       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
424       zkw.keeperException(e);
425       return -1;
426     } catch (InterruptedException e) {
427       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
428       zkw.interruptedException(e);
429       return -1;
430     }
431   }
432 
433   //
434   // Znode listings
435   //
436 
437   /**
438    * Lists the children znodes of the specified znode.  Also sets a watch on
439    * the specified znode which will capture a NodeDeleted event on the specified
440    * znode as well as NodeChildrenChanged if any children of the specified znode
441    * are created or deleted.
442    *
443    * Returns null if the specified node does not exist.  Otherwise returns a
444    * list of children of the specified node.  If the node exists but it has no
445    * children, an empty list will be returned.
446    *
447    * @param zkw zk reference
448    * @param znode path of node to list and watch children of
449    * @return list of children of the specified node, an empty list if the node
450    *          exists but has no children, and null if the node does not exist
451    * @throws KeeperException if unexpected zookeeper exception
452    */
453   public static List<String> listChildrenAndWatchForNewChildren(
454       ZooKeeperWatcher zkw, String znode)
455   throws KeeperException {
456     try {
457       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
458       return children;
459     } catch(KeeperException.NoNodeException ke) {
460       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
461           "because node does not exist (not an error)"));
462       return null;
463     } catch (KeeperException e) {
464       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
465       zkw.keeperException(e);
466       return null;
467     } catch (InterruptedException e) {
468       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
469       zkw.interruptedException(e);
470       return null;
471     }
472   }
473 
474   /**
475    * List all the children of the specified znode, setting a watch for children
476    * changes and also setting a watch on every individual child in order to get
477    * the NodeCreated and NodeDeleted events.
478    * @param zkw zookeeper reference
479    * @param znode node to get children of and watch
480    * @return list of znode names, null if the node doesn't exist
481    * @throws KeeperException
482    */
483   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
484       String znode) throws KeeperException {
485     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
486     if (children == null) {
487       return null;
488     }
489     for (String child : children) {
490       watchAndCheckExists(zkw, joinZNode(znode, child));
491     }
492     return children;
493   }
494 
495   /**
496    * Lists the children of the specified znode without setting any watches.
497    *
498    * Sets no watches at all, this method is best effort.
499    *
500    * Returns an empty list if the node has no children.  Returns null if the
501    * parent node itself does not exist.
502    *
503    * @param zkw zookeeper reference
504    * @param znode node to get children
505    * @return list of data of children of specified znode, empty if no children,
506    *         null if parent does not exist
507    * @throws KeeperException if unexpected zookeeper exception
508    */
509   public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
510   throws KeeperException {
511     List<String> children = null;
512     try {
513       // List the children without watching
514       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
515     } catch(KeeperException.NoNodeException nne) {
516       return null;
517     } catch(InterruptedException ie) {
518       zkw.interruptedException(ie);
519     }
520     return children;
521   }
522 
523   /**
524    * Simple class to hold a node path and node data.
525    * @deprecated Unused
526    */
527   @Deprecated
528   public static class NodeAndData {
529     private String node;
530     private byte [] data;
531     public NodeAndData(String node, byte [] data) {
532       this.node = node;
533       this.data = data;
534     }
535     public String getNode() {
536       return node;
537     }
538     public byte [] getData() {
539       return data;
540     }
541     @Override
542     public String toString() {
543       return node;
544     }
545     public boolean isEmpty() {
546       return (data == null || data.length == 0);
547     }
548   }
549 
550   /**
551    * Checks if the specified znode has any children.  Sets no watches.
552    *
553    * Returns true if the node exists and has children.  Returns false if the
554    * node does not exist or if the node does not have any children.
555    *
556    * Used during master initialization to determine if the master is a
557    * failed-over-to master or the first master during initial cluster startup.
558    * If the directory for regionserver ephemeral nodes is empty then this is
559    * a cluster startup, if not then it is not cluster startup.
560    *
561    * @param zkw zk reference
562    * @param znode path of node to check for children of
563    * @return true if node has children, false if not or node does not exist
564    * @throws KeeperException if unexpected zookeeper exception
565    */
566   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
567   throws KeeperException {
568     try {
569       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
570     } catch(KeeperException.NoNodeException ke) {
571       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
572       "because node does not exist (not an error)"));
573       return false;
574     } catch (KeeperException e) {
575       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
576       zkw.keeperException(e);
577       return false;
578     } catch (InterruptedException e) {
579       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
580       zkw.interruptedException(e);
581       return false;
582     }
583   }
584 
585   /**
586    * Get the number of children of the specified node.
587    *
588    * If the node does not exist or has no children, returns 0.
589    *
590    * Sets no watches at all.
591    *
592    * @param zkw zk reference
593    * @param znode path of node to count children of
594    * @return number of children of specified node, 0 if none or parent does not
595    *         exist
596    * @throws KeeperException if unexpected zookeeper exception
597    */
598   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
599   throws KeeperException {
600     try {
601       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
602       return stat == null ? 0 : stat.getNumChildren();
603     } catch(KeeperException e) {
604       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
605       zkw.keeperException(e);
606     } catch(InterruptedException e) {
607       zkw.interruptedException(e);
608     }
609     return 0;
610   }
611 
612   //
613   // Data retrieval
614   //
615 
616   /**
617    * Get znode data. Does not set a watcher.
618    * @return ZNode data, null if the node does not exist or if there is an
619    *  error.
620    */
621   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
622       throws KeeperException, InterruptedException {
623     try {
624       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
625       logRetrievedMsg(zkw, znode, data, false);
626       return data;
627     } catch (KeeperException.NoNodeException e) {
628       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
629           "because node does not exist (not an error)"));
630       return null;
631     } catch (KeeperException e) {
632       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
633       zkw.keeperException(e);
634       return null;
635     }
636   }
637 
638   /**
639    * Get the data at the specified znode and set a watch.
640    *
641    * Returns the data and sets a watch if the node exists.  Returns null and no
642    * watch is set if the node does not exist or there is an exception.
643    *
644    * @param zkw zk reference
645    * @param znode path of node
646    * @return data of the specified znode, or null
647    * @throws KeeperException if unexpected zookeeper exception
648    */
649   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
650   throws KeeperException {
651     return getDataInternal(zkw, znode, null, true);
652   }
653 
654   /**
655    * Get the data at the specified znode and set a watch.
656    *
657    * Returns the data and sets a watch if the node exists.  Returns null and no
658    * watch is set if the node does not exist or there is an exception.
659    *
660    * @param zkw zk reference
661    * @param znode path of node
662    * @param stat object to populate the version of the znode
663    * @return data of the specified znode, or null
664    * @throws KeeperException if unexpected zookeeper exception
665    */
666   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
667       Stat stat) throws KeeperException {
668     return getDataInternal(zkw, znode, stat, true);
669   }
670 
671   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
672       boolean watcherSet)
673       throws KeeperException {
674     try {
675       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
676       logRetrievedMsg(zkw, znode, data, watcherSet);
677       return data;
678     } catch (KeeperException.NoNodeException e) {
679       // This log can get pretty annoying when we cycle on 100ms waits.
680       // Enable trace if you really want to see it.
681       LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
682         "because node does not exist (not an error)"));
683       return null;
684     } catch (KeeperException e) {
685       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
686       zkw.keeperException(e);
687       return null;
688     } catch (InterruptedException e) {
689       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
690       zkw.interruptedException(e);
691       return null;
692     }
693   }
694 
695   /**
696    * Get the data at the specified znode without setting a watch.
697    *
698    * Returns the data if the node exists.  Returns null if the node does not
699    * exist.
700    *
701    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
702    * not interested.
703    *
704    * @param zkw zk reference
705    * @param znode path of node
706    * @param stat node status to get if node exists
707    * @return data of the specified znode, or null if node does not exist
708    * @throws KeeperException if unexpected zookeeper exception
709    */
710   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
711       Stat stat)
712   throws KeeperException {
713     try {
714       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
715       logRetrievedMsg(zkw, znode, data, false);
716       return data;
717     } catch (KeeperException.NoNodeException e) {
718       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
719           "because node does not exist (not necessarily an error)"));
720       return null;
721     } catch (KeeperException e) {
722       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
723       zkw.keeperException(e);
724       return null;
725     } catch (InterruptedException e) {
726       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
727       zkw.interruptedException(e);
728       return null;
729     }
730   }
731 
732   /**
733    * Returns the date of child znodes of the specified znode.  Also sets a watch on
734    * the specified znode which will capture a NodeDeleted event on the specified
735    * znode as well as NodeChildrenChanged if any children of the specified znode
736    * are created or deleted.
737    *
738    * Returns null if the specified node does not exist.  Otherwise returns a
739    * list of children of the specified node.  If the node exists but it has no
740    * children, an empty list will be returned.
741    *
742    * @param zkw zk reference
743    * @param baseNode path of node to list and watch children of
744    * @return list of data of children of the specified node, an empty list if the node
745    *          exists but has no children, and null if the node does not exist
746    * @throws KeeperException if unexpected zookeeper exception
747    * @deprecated Unused
748    */
749   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
750       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
751     List<String> nodes =
752       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
753     if (nodes != null) {
754       List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
755       for (String node : nodes) {
756         String nodePath = ZKUtil.joinZNode(baseNode, node);
757         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
758         newNodes.add(new NodeAndData(nodePath, data));
759       }
760       return newNodes;
761     }
762     return null;
763   }
764 
765   /**
766    * Update the data of an existing node with the expected version to have the
767    * specified data.
768    *
769    * Throws an exception if there is a version mismatch or some other problem.
770    *
771    * Sets no watches under any conditions.
772    *
773    * @param zkw zk reference
774    * @param znode
775    * @param data
776    * @param expectedVersion
777    * @throws KeeperException if unexpected zookeeper exception
778    * @throws KeeperException.BadVersionException if version mismatch
779    * @deprecated Unused
780    */
781   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
782       byte [] data, int expectedVersion)
783   throws KeeperException {
784     try {
785       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
786     } catch(InterruptedException ie) {
787       zkw.interruptedException(ie);
788     }
789   }
790 
791   //
792   // Data setting
793   //
794 
795   /**
796    * Sets the data of the existing znode to be the specified data.  Ensures that
797    * the current data has the specified expected version.
798    *
799    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
800    *
801    * <p>If their is a version mismatch, method returns null.
802    *
803    * <p>No watches are set but setting data will trigger other watchers of this
804    * node.
805    *
806    * <p>If there is another problem, a KeeperException will be thrown.
807    *
808    * @param zkw zk reference
809    * @param znode path of node
810    * @param data data to set for node
811    * @param expectedVersion version expected when setting data
812    * @return true if data set, false if version mismatch
813    * @throws KeeperException if unexpected zookeeper exception
814    */
815   public static boolean setData(ZooKeeperWatcher zkw, String znode,
816       byte [] data, int expectedVersion)
817   throws KeeperException, KeeperException.NoNodeException {
818     try {
819       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
820     } catch (InterruptedException e) {
821       zkw.interruptedException(e);
822       return false;
823     }
824   }
825 
826   /**
827    * Set data into node creating node if it doesn't yet exist.
828    * Does not set watch.
829    *
830    * @param zkw zk reference
831    * @param znode path of node
832    * @param data data to set for node
833    * @throws KeeperException
834    */
835   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
836       final byte [] data)
837   throws KeeperException {
838     if (checkExists(zkw, znode) == -1) {
839       ZKUtil.createWithParents(zkw, znode, data);
840     } else {
841       ZKUtil.setData(zkw, znode, data);
842     }
843   }
844 
845   /**
846    * Sets the data of the existing znode to be the specified data.  The node
847    * must exist but no checks are done on the existing data or version.
848    *
849    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
850    *
851    * <p>No watches are set but setting data will trigger other watchers of this
852    * node.
853    *
854    * <p>If there is another problem, a KeeperException will be thrown.
855    *
856    * @param zkw zk reference
857    * @param znode path of node
858    * @param data data to set for node
859    * @throws KeeperException if unexpected zookeeper exception
860    */
861   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
862   throws KeeperException, KeeperException.NoNodeException {
863     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
864   }
865 
866   private static void setData(ZooKeeperWatcher zkw, SetData setData)
867   throws KeeperException, KeeperException.NoNodeException {
868     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
869     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
870   }
871 
872   /**
873    * Returns whether or not secure authentication is enabled
874    * (whether <code>hbase.security.authentication</code> is set to
875    * <code>kerberos</code>.
876    */
877   public static boolean isSecureZooKeeper(Configuration conf) {
878     // Detection for embedded HBase client with jaas configuration
879     // defined for third party programs.
880     try {
881       javax.security.auth.login.Configuration testConfig =
882           javax.security.auth.login.Configuration.getConfiguration();
883       if (testConfig.getAppConfigurationEntry("Client") == null
884           && testConfig.getAppConfigurationEntry(
885             JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME) == null
886           && testConfig.getAppConfigurationEntry(
887               JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME) == null) {
888         return false;
889       }
890     } catch(Exception e) {
891       // No Jaas configuration defined.
892       return false;
893     }
894 
895     // Master & RSs uses hbase.zookeeper.client.*
896     return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
897   }
898 
899   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
900     return createACL(zkw, node, isSecureZooKeeper(zkw.getConfiguration()));
901   }
902 
903   public static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node,
904     boolean isSecureZooKeeper) {
905     if (!node.startsWith(zkw.baseZNode)) {
906       return Ids.OPEN_ACL_UNSAFE;
907     }
908     if (isSecureZooKeeper) {
909       ArrayList<ACL> acls = new ArrayList<ACL>();
910       // add permission to hbase supper user
911       String[] superUsers = zkw.getConfiguration().getStrings(Superusers.SUPERUSER_CONF_KEY);
912       if (superUsers != null) {
913         List<String> groups = new ArrayList<String>();
914         for (String user : superUsers) {
915           if (user.startsWith(AuthUtil.GROUP_PREFIX)) {
916             // TODO: Set node ACL for groups when ZK supports this feature
917             groups.add(user);
918           } else {
919             acls.add(new ACL(Perms.ALL, new Id("sasl", user)));
920           }
921         }
922         if (!groups.isEmpty()) {
923           LOG.warn("Znode ACL setting for group " + groups
924               + " is skipped, Zookeeper doesn't support this feature presently.");
925         }
926       }
927       // Certain znodes are accessed directly by the client,
928       // so they must be readable by non-authenticated clients
929       if (zkw.isClientReadable(node)) {
930         acls.addAll(Ids.CREATOR_ALL_ACL);
931         acls.addAll(Ids.READ_ACL_UNSAFE);
932       } else {
933         acls.addAll(Ids.CREATOR_ALL_ACL);
934       }
935       return acls;
936     } else {
937       return Ids.OPEN_ACL_UNSAFE;
938     }
939   }
940 
941   //
942   // Node creation
943   //
944 
945   /**
946    *
947    * Set the specified znode to be an ephemeral node carrying the specified
948    * data.
949    *
950    * If the node is created successfully, a watcher is also set on the node.
951    *
952    * If the node is not created successfully because it already exists, this
953    * method will also set a watcher on the node.
954    *
955    * If there is another problem, a KeeperException will be thrown.
956    *
957    * @param zkw zk reference
958    * @param znode path of node
959    * @param data data of node
960    * @return true if node created, false if not, watch set in both cases
961    * @throws KeeperException if unexpected zookeeper exception
962    */
963   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
964       String znode, byte [] data)
965   throws KeeperException {
966     boolean ret = true;
967     try {
968       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
969           CreateMode.EPHEMERAL);
970     } catch (KeeperException.NodeExistsException nee) {
971       ret = false;
972     } catch (InterruptedException e) {
973       LOG.info("Interrupted", e);
974       Thread.currentThread().interrupt();
975     }
976     if(!watchAndCheckExists(zkw, znode)) {
977       // It did exist but now it doesn't, try again
978       return createEphemeralNodeAndWatch(zkw, znode, data);
979     }
980     return ret;
981   }
982 
983   /**
984    * Creates the specified znode to be a persistent node carrying the specified
985    * data.
986    *
987    * Returns true if the node was successfully created, false if the node
988    * already existed.
989    *
990    * If the node is created successfully, a watcher is also set on the node.
991    *
992    * If the node is not created successfully because it already exists, this
993    * method will also set a watcher on the node but return false.
994    *
995    * If there is another problem, a KeeperException will be thrown.
996    *
997    * @param zkw zk reference
998    * @param znode path of node
999    * @param data data of node
1000    * @return true if node created, false if not, watch set in both cases
1001    * @throws KeeperException if unexpected zookeeper exception
1002    */
1003   public static boolean createNodeIfNotExistsAndWatch(
1004       ZooKeeperWatcher zkw, String znode, byte [] data)
1005   throws KeeperException {
1006     boolean ret = true;
1007     try {
1008       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1009           CreateMode.PERSISTENT);
1010     } catch (KeeperException.NodeExistsException nee) {
1011       ret = false;
1012     } catch (InterruptedException e) {
1013       zkw.interruptedException(e);
1014       return false;
1015     }
1016     try {
1017       zkw.getRecoverableZooKeeper().exists(znode, zkw);
1018     } catch (InterruptedException e) {
1019       zkw.interruptedException(e);
1020       return false;
1021     }
1022     return ret;
1023   }
1024 
1025   /**
1026    * Creates the specified znode with the specified data but does not watch it.
1027    *
1028    * Returns the znode of the newly created node
1029    *
1030    * If there is another problem, a KeeperException will be thrown.
1031    *
1032    * @param zkw zk reference
1033    * @param znode path of node
1034    * @param data data of node
1035    * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1036    * @return true name of the newly created znode or null
1037    * @throws KeeperException if unexpected zookeeper exception
1038    */
1039   public static String createNodeIfNotExistsNoWatch(ZooKeeperWatcher zkw, String znode,
1040       byte[] data, CreateMode createMode) throws KeeperException {
1041 
1042     String createdZNode = null;
1043     try {
1044       createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1045           createACL(zkw, znode), createMode);
1046     } catch (KeeperException.NodeExistsException nee) {
1047       return znode;
1048     } catch (InterruptedException e) {
1049       zkw.interruptedException(e);
1050       return null;
1051     }
1052     return createdZNode;
1053   }
1054 
1055   /**
1056    * Creates the specified node with the specified data and watches it.
1057    *
1058    * <p>Throws an exception if the node already exists.
1059    *
1060    * <p>The node created is persistent and open access.
1061    *
1062    * <p>Returns the version number of the created node if successful.
1063    *
1064    * @param zkw zk reference
1065    * @param znode path of node to create
1066    * @param data data of node to create
1067    * @return version of node created
1068    * @throws KeeperException if unexpected zookeeper exception
1069    * @throws KeeperException.NodeExistsException if node already exists
1070    */
1071   public static int createAndWatch(ZooKeeperWatcher zkw,
1072       String znode, byte [] data)
1073   throws KeeperException, KeeperException.NodeExistsException {
1074     try {
1075       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1076           CreateMode.PERSISTENT);
1077       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1078       if (stat == null){
1079         // Likely a race condition. Someone deleted the znode.
1080         throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1081             "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1082       }
1083      return stat.getVersion();
1084     } catch (InterruptedException e) {
1085       zkw.interruptedException(e);
1086       return -1;
1087     }
1088   }
1089 
1090   /**
1091    * Async creates the specified node with the specified data.
1092    *
1093    * <p>Throws an exception if the node already exists.
1094    *
1095    * <p>The node created is persistent and open access.
1096    *
1097    * @param zkw zk reference
1098    * @param znode path of node to create
1099    * @param data data of node to create
1100    * @param cb
1101    * @param ctx
1102    * @throws KeeperException if unexpected zookeeper exception
1103    * @throws KeeperException.NodeExistsException if node already exists
1104    */
1105   public static void asyncCreate(ZooKeeperWatcher zkw,
1106       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1107       final Object ctx) {
1108     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1109         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1110   }
1111 
1112   /**
1113    * Creates the specified node, iff the node does not exist.  Does not set a
1114    * watch and fails silently if the node already exists.
1115    *
1116    * The node created is persistent and open access.
1117    *
1118    * @param zkw zk reference
1119    * @param znode path of node
1120    * @throws KeeperException if unexpected zookeeper exception
1121    */
1122   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1123       String znode) throws KeeperException {
1124     createAndFailSilent(zkw, znode, new byte[0]);
1125   }
1126 
1127   /**
1128    * Creates the specified node containing specified data, iff the node does not exist.  Does
1129    * not set a watch and fails silently if the node already exists.
1130    *
1131    * The node created is persistent and open access.
1132    *
1133    * @param zkw zk reference
1134    * @param znode path of node
1135    * @param data a byte array data to store in the znode
1136    * @throws KeeperException if unexpected zookeeper exception
1137    */
1138   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1139       String znode, byte[] data)
1140   throws KeeperException {
1141     createAndFailSilent(zkw,
1142         (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1143   }
1144 
1145   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1146   throws KeeperException {
1147     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1148     String znode = create.getPath();
1149     try {
1150       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1151       if (zk.exists(znode, false) == null) {
1152         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1153       }
1154     } catch(KeeperException.NodeExistsException nee) {
1155     } catch(KeeperException.NoAuthException nee){
1156       try {
1157         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1158           // If we failed to create the file and it does not already exist.
1159           throw(nee);
1160         }
1161       } catch (InterruptedException ie) {
1162         zkw.interruptedException(ie);
1163       }
1164     } catch(InterruptedException ie) {
1165       zkw.interruptedException(ie);
1166     }
1167   }
1168 
1169   /**
1170    * Creates the specified node and all parent nodes required for it to exist.
1171    *
1172    * No watches are set and no errors are thrown if the node already exists.
1173    *
1174    * The nodes created are persistent and open access.
1175    *
1176    * @param zkw zk reference
1177    * @param znode path of node
1178    * @throws KeeperException if unexpected zookeeper exception
1179    */
1180   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1181   throws KeeperException {
1182     createWithParents(zkw, znode, new byte[0]);
1183   }
1184 
1185   /**
1186    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1187    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1188    * when the leaf node is created.
1189    *
1190    * No watches are set and no errors are thrown if the node already exists.
1191    *
1192    * The nodes created are persistent and open access.
1193    *
1194    * @param zkw zk reference
1195    * @param znode path of node
1196    * @throws KeeperException if unexpected zookeeper exception
1197    */
1198   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1199   throws KeeperException {
1200     try {
1201       if(znode == null) {
1202         return;
1203       }
1204       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1205           CreateMode.PERSISTENT);
1206     } catch(KeeperException.NodeExistsException nee) {
1207       return;
1208     } catch(KeeperException.NoNodeException nne) {
1209       createWithParents(zkw, getParent(znode));
1210       createWithParents(zkw, znode, data);
1211     } catch(InterruptedException ie) {
1212       zkw.interruptedException(ie);
1213     }
1214   }
1215 
1216   //
1217   // Deletes
1218   //
1219 
1220   /**
1221    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1222    */
1223   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1224   throws KeeperException {
1225     deleteNode(zkw, node, -1);
1226   }
1227 
1228   /**
1229    * Delete the specified node with the specified version.  Sets no watches.
1230    * Throws all exceptions.
1231    */
1232   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1233       int version)
1234   throws KeeperException {
1235     try {
1236       zkw.getRecoverableZooKeeper().delete(node, version);
1237       return true;
1238     } catch(KeeperException.BadVersionException bve) {
1239       return false;
1240     } catch(InterruptedException ie) {
1241       zkw.interruptedException(ie);
1242       return false;
1243     }
1244   }
1245 
1246   /**
1247    * Deletes the specified node.  Fails silent if the node does not exist.
1248    * @param zkw
1249    * @param node
1250    * @throws KeeperException
1251    */
1252   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1253   throws KeeperException {
1254     deleteNodeFailSilent(zkw,
1255       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1256   }
1257 
1258   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1259       DeleteNodeFailSilent dnfs) throws KeeperException {
1260     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1261     try {
1262       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1263     } catch(KeeperException.NoNodeException nne) {
1264     } catch(InterruptedException ie) {
1265       zkw.interruptedException(ie);
1266     }
1267   }
1268 
1269 
1270   /**
1271    * Delete the specified node and all of it's children.
1272    * <p>
1273    * If the node does not exist, just returns.
1274    * <p>
1275    * Sets no watches. Throws all exceptions besides dealing with deletion of
1276    * children.
1277    */
1278   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1279   throws KeeperException {
1280     deleteNodeRecursivelyMultiOrSequential(zkw, true, node);
1281   }
1282 
1283   /**
1284    * Delete all the children of the specified node but not the node itself.
1285    *
1286    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1287    * children.
1288    *
1289    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1290    * Otherwise, run the list of operations sequentially.
1291    *
1292    * @throws KeeperException
1293    */
1294   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1295       throws KeeperException {
1296     deleteChildrenRecursivelyMultiOrSequential(zkw, true, node);
1297   }
1298 
1299   /**
1300    * Delete all the children of the specified node but not the node itself. This
1301    * will first traverse the znode tree for listing the children and then delete
1302    * these znodes using multi-update api or sequential based on the specified
1303    * configurations.
1304    * <p>
1305    * Sets no watches. Throws all exceptions besides dealing with deletion of
1306    * children.
1307    * <p>
1308    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1309    * functionality. Otherwise, run the list of operations sequentially.
1310    * <p>
1311    * If all of the following are true:
1312    * <ul>
1313    * <li>runSequentialOnMultiFailure is true
1314    * <li>hbase.zookeeper.useMulti is true
1315    * </ul>
1316    * on calling multi, we get a ZooKeeper exception that can be handled by a
1317    * sequential call(*), we retry the operations one-by-one (sequentially).
1318    *
1319    * @param zkw
1320    *          - zk reference
1321    * @param runSequentialOnMultiFailure
1322    *          - if true when we get a ZooKeeper exception that could retry the
1323    *          operations one-by-one (sequentially)
1324    * @param pathRoots
1325    *          - path of the parent node(s)
1326    * @throws KeeperException.NotEmptyException
1327    *           if node has children while deleting
1328    * @throws KeeperException
1329    *           if unexpected ZooKeeper exception
1330    * @throws IllegalArgumentException
1331    *           if an invalid path is specified
1332    */
1333   public static void deleteChildrenRecursivelyMultiOrSequential(
1334       ZooKeeperWatcher zkw, boolean runSequentialOnMultiFailure,
1335       String... pathRoots) throws KeeperException {
1336     if (pathRoots == null || pathRoots.length <= 0) {
1337       LOG.warn("Given path is not valid!");
1338       return;
1339     }
1340     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1341     for (String eachRoot : pathRoots) {
1342       List<String> children = listChildrenBFSNoWatch(zkw, eachRoot);
1343       // Delete the leaves first and eventually get rid of the root
1344       for (int i = children.size() - 1; i >= 0; --i) {
1345         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1346       }
1347     }
1348     // atleast one element should exist
1349     if (ops.size() > 0) {
1350       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1351     }
1352   }
1353 
1354   /**
1355    * Delete the specified node and its children. This traverse the
1356    * znode tree for listing the children and then delete
1357    * these znodes including the parent using multi-update api or
1358    * sequential based on the specified configurations.
1359    * <p>
1360    * Sets no watches. Throws all exceptions besides dealing with deletion of
1361    * children.
1362    * <p>
1363    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1364    * functionality. Otherwise, run the list of operations sequentially.
1365    * <p>
1366    * If all of the following are true:
1367    * <ul>
1368    * <li>runSequentialOnMultiFailure is true
1369    * <li>hbase.zookeeper.useMulti is true
1370    * </ul>
1371    * on calling multi, we get a ZooKeeper exception that can be handled by a
1372    * sequential call(*), we retry the operations one-by-one (sequentially).
1373    *
1374    * @param zkw
1375    *          - zk reference
1376    * @param runSequentialOnMultiFailure
1377    *          - if true when we get a ZooKeeper exception that could retry the
1378    *          operations one-by-one (sequentially)
1379    * @param pathRoots
1380    *          - path of the parent node(s)
1381    * @throws KeeperException.NotEmptyException
1382    *           if node has children while deleting
1383    * @throws KeeperException
1384    *           if unexpected ZooKeeper exception
1385    * @throws IllegalArgumentException
1386    *           if an invalid path is specified
1387    */
1388   public static void deleteNodeRecursivelyMultiOrSequential(ZooKeeperWatcher zkw,
1389       boolean runSequentialOnMultiFailure, String... pathRoots) throws KeeperException {
1390     if (pathRoots == null || pathRoots.length <= 0) {
1391       LOG.warn("Given path is not valid!");
1392       return;
1393     }
1394     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1395     for (String eachRoot : pathRoots) {
1396       // Zookeeper Watches are one time triggers; When children of parent nodes are deleted
1397       // recursively, must set another watch, get notified of delete node
1398       List<String> children = listChildrenBFSAndWatchThem(zkw, eachRoot);
1399       // Delete the leaves first and eventually get rid of the root
1400       for (int i = children.size() - 1; i >= 0; --i) {
1401         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1402       }
1403       try {
1404         if (zkw.getRecoverableZooKeeper().exists(eachRoot, zkw) != null) {
1405           ops.add(ZKUtilOp.deleteNodeFailSilent(eachRoot));
1406         }
1407       } catch (InterruptedException e) {
1408         zkw.interruptedException(e);
1409       }
1410     }
1411     // atleast one element should exist
1412     if (ops.size() > 0) {
1413       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1414     }
1415   }
1416 
1417   /**
1418    * BFS Traversal of all the children under path, with the entries in the list,
1419    * in the same order as that of the traversal. Lists all the children without
1420    * setting any watches.
1421    *
1422    * @param zkw
1423    *          - zk reference
1424    * @param znode
1425    *          - path of node
1426    * @return list of children znodes under the path
1427    * @throws KeeperException
1428    *           if unexpected ZooKeeper exception
1429    */
1430   private static List<String> listChildrenBFSNoWatch(ZooKeeperWatcher zkw,
1431       final String znode) throws KeeperException {
1432     Deque<String> queue = new LinkedList<String>();
1433     List<String> tree = new ArrayList<String>();
1434     queue.add(znode);
1435     while (true) {
1436       String node = queue.pollFirst();
1437       if (node == null) {
1438         break;
1439       }
1440       List<String> children = listChildrenNoWatch(zkw, node);
1441       if (children == null) {
1442         continue;
1443       }
1444       for (final String child : children) {
1445         final String childPath = node + "/" + child;
1446         queue.add(childPath);
1447         tree.add(childPath);
1448       }
1449     }
1450     return tree;
1451   }
1452 
1453   /**
1454    * BFS Traversal of all the children under path, with the entries in the list,
1455    * in the same order as that of the traversal.
1456    * Lists all the children and set watches on to them.
1457    *
1458    * @param zkw
1459    *          - zk reference
1460    * @param znode
1461    *          - path of node
1462    * @return list of children znodes under the path
1463    * @throws KeeperException
1464    *           if unexpected ZooKeeper exception
1465    */
1466   private static List<String> listChildrenBFSAndWatchThem(ZooKeeperWatcher zkw, final String znode)
1467       throws KeeperException {
1468     Deque<String> queue = new LinkedList<String>();
1469     List<String> tree = new ArrayList<String>();
1470     queue.add(znode);
1471     while (true) {
1472       String node = queue.pollFirst();
1473       if (node == null) {
1474         break;
1475       }
1476       List<String> children = listChildrenAndWatchThem(zkw, node);
1477       if (children == null) {
1478         continue;
1479       }
1480       for (final String child : children) {
1481         final String childPath = node + "/" + child;
1482         queue.add(childPath);
1483         tree.add(childPath);
1484       }
1485     }
1486     return tree;
1487   }
1488 
1489   /**
1490    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1491    * These actions are higher-level than ZKOp actions, which represent
1492    * individual actions in the ZooKeeper API, like create.
1493    */
1494   public abstract static class ZKUtilOp {
1495     private String path;
1496 
1497     private ZKUtilOp(String path) {
1498       this.path = path;
1499     }
1500 
1501     /**
1502      * @return a createAndFailSilent ZKUtilOp
1503      */
1504     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1505       return new CreateAndFailSilent(path, data);
1506     }
1507 
1508     /**
1509      * @return a deleteNodeFailSilent ZKUtilOP
1510      */
1511     public static ZKUtilOp deleteNodeFailSilent(String path) {
1512       return new DeleteNodeFailSilent(path);
1513     }
1514 
1515     /**
1516      * @return a setData ZKUtilOp
1517      */
1518     public static ZKUtilOp setData(String path, byte [] data) {
1519       return new SetData(path, data);
1520     }
1521 
1522     /**
1523      * @return path to znode where the ZKOp will occur
1524      */
1525     public String getPath() {
1526       return path;
1527     }
1528 
1529     /**
1530      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1531      * (attempt to create node, ignore error if already exists)
1532      */
1533     public static class CreateAndFailSilent extends ZKUtilOp {
1534       private byte [] data;
1535 
1536       private CreateAndFailSilent(String path, byte [] data) {
1537         super(path);
1538         this.data = data;
1539       }
1540 
1541       public byte[] getData() {
1542         return data;
1543       }
1544 
1545       @Override
1546       public boolean equals(Object o) {
1547         if (this == o) return true;
1548         if (!(o instanceof CreateAndFailSilent)) return false;
1549 
1550         CreateAndFailSilent op = (CreateAndFailSilent) o;
1551         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1552       }
1553 
1554       @Override
1555       public int hashCode() {
1556         int ret = 17 + getPath().hashCode() * 31;
1557         return ret * 31 + Bytes.hashCode(data);
1558       }
1559     }
1560 
1561     /**
1562      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1563      * (attempt to delete node, ignore error if node doesn't exist)
1564      */
1565     public static class DeleteNodeFailSilent extends ZKUtilOp {
1566       private DeleteNodeFailSilent(String path) {
1567         super(path);
1568       }
1569 
1570       @Override
1571       public boolean equals(Object o) {
1572         if (this == o) return true;
1573         if (!(o instanceof DeleteNodeFailSilent)) return false;
1574 
1575         return super.equals(o);
1576       }
1577 
1578       @Override
1579       public int hashCode() {
1580         return getPath().hashCode();
1581       }
1582     }
1583 
1584     /**
1585      * ZKUtilOp representing setData in ZooKeeper
1586      */
1587     public static class SetData extends ZKUtilOp {
1588       private byte [] data;
1589 
1590       private SetData(String path, byte [] data) {
1591         super(path);
1592         this.data = data;
1593       }
1594 
1595       public byte[] getData() {
1596         return data;
1597       }
1598 
1599       @Override
1600       public boolean equals(Object o) {
1601         if (this == o) return true;
1602         if (!(o instanceof SetData)) return false;
1603 
1604         SetData op = (SetData) o;
1605         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1606       }
1607 
1608       @Override
1609       public int hashCode() {
1610         int ret = getPath().hashCode();
1611         return ret * 31 + Bytes.hashCode(data);
1612       }
1613     }
1614   }
1615 
1616   /**
1617    * Convert from ZKUtilOp to ZKOp
1618    */
1619   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1620   throws UnsupportedOperationException {
1621     if(op == null) return null;
1622 
1623     if (op instanceof CreateAndFailSilent) {
1624       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1625       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1626         CreateMode.PERSISTENT);
1627     } else if (op instanceof DeleteNodeFailSilent) {
1628       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1629       return Op.delete(dnfs.getPath(), -1);
1630     } else if (op instanceof SetData) {
1631       SetData sd = (SetData)op;
1632       return Op.setData(sd.getPath(), sd.getData(), -1);
1633     } else {
1634       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1635         + op.getClass().getName());
1636     }
1637   }
1638 
1639   /**
1640    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1641    * Otherwise, run the list of operations sequentially.
1642    *
1643    * If all of the following are true:
1644    * - runSequentialOnMultiFailure is true
1645    * - hbase.zookeeper.useMulti is true
1646    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1647    * Then:
1648    * - we retry the operations one-by-one (sequentially)
1649    *
1650    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1651    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1652    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1653    * This will cause all operations in the multi to fail, however, because
1654    * the NodeExistsException that zk.create throws will fail the multi transaction.
1655    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1656    * result in the correct final state, but means that the operations will not run atomically.
1657    *
1658    * @throws KeeperException
1659    */
1660   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1661       boolean runSequentialOnMultiFailure) throws KeeperException {
1662     if (ops == null) return;
1663     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1664 
1665     if (useMulti) {
1666       List<Op> zkOps = new LinkedList<Op>();
1667       for (ZKUtilOp op : ops) {
1668         zkOps.add(toZooKeeperOp(zkw, op));
1669       }
1670       try {
1671         zkw.getRecoverableZooKeeper().multi(zkOps);
1672       } catch (KeeperException ke) {
1673        switch (ke.code()) {
1674          case NODEEXISTS:
1675          case NONODE:
1676          case BADVERSION:
1677          case NOAUTH:
1678            // if we get an exception that could be solved by running sequentially
1679            // (and the client asked us to), then break out and run sequentially
1680            if (runSequentialOnMultiFailure) {
1681              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1682                + "  Attempting to run operations sequentially because"
1683                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1684              processSequentially(zkw, ops);
1685              break;
1686            }
1687           default:
1688             throw ke;
1689         }
1690       } catch (InterruptedException ie) {
1691         zkw.interruptedException(ie);
1692       }
1693     } else {
1694       // run sequentially
1695       processSequentially(zkw, ops);
1696     }
1697 
1698   }
1699 
1700   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1701       throws KeeperException, NoNodeException {
1702     for (ZKUtilOp op : ops) {
1703       if (op instanceof CreateAndFailSilent) {
1704         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1705       } else if (op instanceof DeleteNodeFailSilent) {
1706         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1707       } else if (op instanceof SetData) {
1708         setData(zkw, (SetData) op);
1709       } else {
1710         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1711             + op.getClass().getName());
1712       }
1713     }
1714   }
1715 
1716   //
1717   // ZooKeeper cluster information
1718   //
1719 
1720   /** @return String dump of everything in ZooKeeper. */
1721   public static String dump(ZooKeeperWatcher zkw) {
1722     StringBuilder sb = new StringBuilder();
1723     try {
1724       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1725       sb.append("\nActive master address: ");
1726       try {
1727         sb.append(MasterAddressTracker.getMasterAddress(zkw));
1728       } catch (IOException e) {
1729         sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1730       }
1731       sb.append("\nBackup master addresses:");
1732       for (String child : listChildrenNoWatch(zkw,
1733                                               zkw.backupMasterAddressesZNode)) {
1734         sb.append("\n ").append(child);
1735       }
1736       sb.append("\nRegion server holding hbase:meta: "
1737         + new MetaTableLocator().getMetaRegionLocation(zkw));
1738       Configuration conf = HBaseConfiguration.create();
1739       int numMetaReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
1740                HConstants.DEFAULT_META_REPLICA_NUM);
1741       for (int i = 1; i < numMetaReplicas; i++) {
1742         sb.append("\nRegion server holding hbase:meta, replicaId " + i + " "
1743                     + new MetaTableLocator().getMetaRegionLocation(zkw, i));
1744       }
1745       sb.append("\nRegion servers:");
1746       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1747         sb.append("\n ").append(child);
1748       }
1749       try {
1750         getReplicationZnodesDump(zkw, sb);
1751       } catch (KeeperException ke) {
1752         LOG.warn("Couldn't get the replication znode dump", ke);
1753       }
1754       sb.append("\nQuorum Server Statistics:");
1755       String[] servers = zkw.getQuorum().split(",");
1756       for (String server : servers) {
1757         sb.append("\n ").append(server);
1758         try {
1759           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1760 
1761           if (stat == null) {
1762             sb.append("[Error] invalid quorum server: " + server);
1763             break;
1764           }
1765 
1766           for (String s : stat) {
1767             sb.append("\n  ").append(s);
1768           }
1769         } catch (Exception e) {
1770           sb.append("\n  ERROR: ").append(e.getMessage());
1771         }
1772       }
1773     } catch (KeeperException ke) {
1774       sb.append("\nFATAL ZooKeeper Exception!\n");
1775       sb.append("\n" + ke.getMessage());
1776     }
1777     return sb.toString();
1778   }
1779 
1780   /**
1781    * Appends replication znodes to the passed StringBuilder.
1782    * @param zkw
1783    * @param sb
1784    * @throws KeeperException
1785    */
1786   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1787       throws KeeperException {
1788     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1789       "replication");
1790     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1791     if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
1792     // do a ls -r on this znode
1793     sb.append("\n").append(replicationZnode).append(": ");
1794     List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
1795     for (String child : children) {
1796       String znode = joinZNode(replicationZnode, child);
1797       if (child.equals(zkw.getConfiguration().get("zookeeper.znode.replication.peers", "peers"))) {
1798         appendPeersZnodes(zkw, znode, sb);
1799       } else if (child.equals(zkw.getConfiguration().
1800           get("zookeeper.znode.replication.rs", "rs"))) {
1801         appendRSZnodes(zkw, znode, sb);
1802       } else if (child.equals(zkw.getConfiguration().get(
1803         ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
1804         ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT))) {
1805         appendHFileRefsZnodes(zkw, znode, sb);
1806       }
1807     }
1808   }
1809 
1810   private static void appendHFileRefsZnodes(ZooKeeperWatcher zkw, String hfileRefsZnode,
1811       StringBuilder sb) throws KeeperException {
1812     sb.append("\n").append(hfileRefsZnode).append(": ");
1813     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, hfileRefsZnode)) {
1814       String znodeToProcess = ZKUtil.joinZNode(hfileRefsZnode, peerIdZnode);
1815       sb.append("\n").append(znodeToProcess).append(": ");
1816       List<String> peerHFileRefsZnodes = ZKUtil.listChildrenNoWatch(zkw, znodeToProcess);
1817       int size = peerHFileRefsZnodes.size();
1818       for (int i = 0; i < size; i++) {
1819         sb.append(peerHFileRefsZnodes.get(i));
1820         if (i != size - 1) {
1821           sb.append(", ");
1822         }
1823       }
1824     }
1825   }
1826 
1827   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, StringBuilder sb)
1828       throws KeeperException {
1829     List<String> stack = new LinkedList<String>();
1830     stack.add(znode);
1831     do {
1832       String znodeToProcess = stack.remove(stack.size() - 1);
1833       sb.append("\n").append(znodeToProcess).append(": ");
1834       byte[] data;
1835       try {
1836         data = ZKUtil.getData(zkw, znodeToProcess);
1837       } catch (InterruptedException e) {
1838         zkw.interruptedException(e);
1839         return;
1840       }
1841       if (data != null && data.length > 0) { // log position
1842         long position = 0;
1843         try {
1844           position = ZKUtil.parseWALPositionFrom(ZKUtil.getData(zkw, znodeToProcess));
1845           sb.append(position);
1846         } catch (DeserializationException ignored) {
1847         } catch (InterruptedException e) {
1848           zkw.interruptedException(e);
1849           return;
1850         }
1851       }
1852       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1853         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1854       }
1855     } while (stack.size() > 0);
1856   }
1857 
1858   private static void appendPeersZnodes(ZooKeeperWatcher zkw, String peersZnode,
1859     StringBuilder sb) throws KeeperException {
1860     int pblen = ProtobufUtil.lengthOfPBMagic();
1861     sb.append("\n").append(peersZnode).append(": ");
1862     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, peersZnode)) {
1863       String znodeToProcess = ZKUtil.joinZNode(peersZnode, peerIdZnode);
1864       byte[] data;
1865       try {
1866         data = ZKUtil.getData(zkw, znodeToProcess);
1867       } catch (InterruptedException e) {
1868         zkw.interruptedException(e);
1869         return;
1870       }
1871       // parse the data of the above peer znode.
1872       try {
1873         ZooKeeperProtos.ReplicationPeer.Builder builder =
1874           ZooKeeperProtos.ReplicationPeer.newBuilder();
1875         ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
1876         String clusterKey = builder.getClusterkey();
1877         sb.append("\n").append(znodeToProcess).append(": ").append(clusterKey);
1878         // add the peer-state.
1879         appendPeerState(zkw, znodeToProcess, sb);
1880       } catch (IOException ipbe) {
1881         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1882       }
1883     }
1884   }
1885 
1886   private static void appendPeerState(ZooKeeperWatcher zkw, String znodeToProcess,
1887       StringBuilder sb) throws KeeperException, InvalidProtocolBufferException {
1888     String peerState = zkw.getConfiguration().get("zookeeper.znode.replication.peers.state",
1889       "peer-state");
1890     int pblen = ProtobufUtil.lengthOfPBMagic();
1891     for (String child : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1892       if (!child.equals(peerState)) continue;
1893       String peerStateZnode = ZKUtil.joinZNode(znodeToProcess, child);
1894       sb.append("\n").append(peerStateZnode).append(": ");
1895       byte[] peerStateData;
1896       try {
1897         peerStateData = ZKUtil.getData(zkw, peerStateZnode);
1898         ZooKeeperProtos.ReplicationState.Builder builder =
1899             ZooKeeperProtos.ReplicationState.newBuilder();
1900         ProtobufUtil.mergeFrom(builder, peerStateData, pblen, peerStateData.length - pblen);
1901         sb.append(builder.getState().name());
1902       } catch (IOException ipbe) {
1903         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1904       } catch (InterruptedException e) {
1905         zkw.interruptedException(e);
1906         return;
1907       }
1908     }
1909   }
1910 
1911   /**
1912    * Gets the statistics from the given server.
1913    *
1914    * @param server  The server to get the statistics from.
1915    * @param timeout  The socket timeout to use.
1916    * @return The array of response strings.
1917    * @throws IOException When the socket communication fails.
1918    */
1919   public static String[] getServerStats(String server, int timeout)
1920   throws IOException {
1921     String[] sp = server.split(":");
1922     if (sp == null || sp.length == 0) {
1923       return null;
1924     }
1925 
1926     String host = sp[0];
1927     int port = sp.length > 1 ? Integer.parseInt(sp[1])
1928         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1929 
1930     Socket socket = new Socket();
1931     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1932     socket.connect(sockAddr, timeout);
1933 
1934     socket.setSoTimeout(timeout);
1935     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1936     BufferedReader in = new BufferedReader(new InputStreamReader(
1937       socket.getInputStream()));
1938     out.println("stat");
1939     out.flush();
1940     ArrayList<String> res = new ArrayList<String>();
1941     while (true) {
1942       String line = in.readLine();
1943       if (line != null) {
1944         res.add(line);
1945       } else {
1946         break;
1947       }
1948     }
1949     socket.close();
1950     return res.toArray(new String[res.size()]);
1951   }
1952 
1953   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1954       final String znode, final byte [] data, final boolean watcherSet) {
1955     if (!LOG.isTraceEnabled()) return;
1956     LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1957       " byte(s) of data from znode " + znode +
1958       (watcherSet? " and set watcher; ": "; data=") +
1959       (data == null? "null": data.length == 0? "empty": (
1960           znode.startsWith(zkw.assignmentZNode)?
1961             ZKAssign.toString(data): // We should not be doing this reaching into another class
1962           znode.startsWith(ZooKeeperWatcher.META_ZNODE_PREFIX)?
1963             getServerNameOrEmptyString(data):
1964           znode.startsWith(zkw.backupMasterAddressesZNode)?
1965             getServerNameOrEmptyString(data):
1966           StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1967   }
1968 
1969   private static String getServerNameOrEmptyString(final byte [] data) {
1970     try {
1971       return ServerName.parseFrom(data).toString();
1972     } catch (DeserializationException e) {
1973       return "";
1974     }
1975   }
1976 
1977   /**
1978    * Waits for HBase installation's base (parent) znode to become available.
1979    * @throws IOException on ZK errors
1980    */
1981   public static void waitForBaseZNode(Configuration conf) throws IOException {
1982     LOG.info("Waiting until the base znode is available");
1983     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1984         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1985     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1986         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1987         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1988 
1989     final int maxTimeMs = 10000;
1990     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1991 
1992     KeeperException keeperEx = null;
1993     try {
1994       try {
1995         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1996           try {
1997             if (zk.exists(parentZNode, false) != null) {
1998               LOG.info("Parent znode exists: " + parentZNode);
1999               keeperEx = null;
2000               break;
2001             }
2002           } catch (KeeperException e) {
2003             keeperEx = e;
2004           }
2005           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
2006         }
2007       } finally {
2008         zk.close();
2009       }
2010     } catch (InterruptedException ex) {
2011       Thread.currentThread().interrupt();
2012     }
2013 
2014     if (keeperEx != null) {
2015       throw new IOException(keeperEx);
2016     }
2017   }
2018 
2019 
2020   public static byte[] blockUntilAvailable(
2021     final ZooKeeperWatcher zkw, final String znode, final long timeout)
2022     throws InterruptedException {
2023     if (timeout < 0) throw new IllegalArgumentException();
2024     if (zkw == null) throw new IllegalArgumentException();
2025     if (znode == null) throw new IllegalArgumentException();
2026 
2027     byte[] data = null;
2028     boolean finished = false;
2029     final long endTime = System.currentTimeMillis() + timeout;
2030     while (!finished) {
2031       try {
2032         data = ZKUtil.getData(zkw, znode);
2033       } catch(KeeperException e) {
2034         if (e instanceof KeeperException.SessionExpiredException
2035             || e instanceof KeeperException.AuthFailedException) {
2036           // non-recoverable errors so stop here
2037           throw new InterruptedException("interrupted due to " + e);
2038         }
2039         LOG.warn("Unexpected exception handling blockUntilAvailable", e);
2040       }
2041 
2042       if (data == null && (System.currentTimeMillis() +
2043         HConstants.SOCKET_RETRY_WAIT_MS < endTime)) {
2044         Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
2045       } else {
2046         finished = true;
2047       }
2048     }
2049 
2050     return data;
2051   }
2052 
2053 
2054   /**
2055    * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
2056    * Used when can't let a {@link DeserializationException} out w/o changing public API.
2057    * @param e Exception to convert
2058    * @return Converted exception
2059    */
2060   public static KeeperException convert(final DeserializationException e) {
2061     KeeperException ke = new KeeperException.DataInconsistencyException();
2062     ke.initCause(e);
2063     return ke;
2064   }
2065 
2066   /**
2067    * Recursively print the current state of ZK (non-transactional)
2068    * @param root name of the root directory in zk to print
2069    * @throws KeeperException
2070    */
2071   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
2072     if (!LOG.isDebugEnabled()) return;
2073     LOG.debug("Current zk system:");
2074     String prefix = "|-";
2075     LOG.debug(prefix + root);
2076     try {
2077       logZKTree(zkw, root, prefix);
2078     } catch (KeeperException e) {
2079       throw new RuntimeException(e);
2080     }
2081   }
2082 
2083   /**
2084    * Helper method to print the current state of the ZK tree.
2085    * @see #logZKTree(ZooKeeperWatcher, String)
2086    * @throws KeeperException if an unexpected exception occurs
2087    */
2088   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix)
2089       throws KeeperException {
2090     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
2091     if (children == null) return;
2092     for (String child : children) {
2093       LOG.debug(prefix + child);
2094       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
2095       logZKTree(zkw, node, prefix + "---");
2096     }
2097   }
2098 
2099   /**
2100    * @param position
2101    * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
2102    *         for use as content of an wal position in a replication queue.
2103    */
2104   public static byte[] positionToByteArray(final long position) {
2105     byte[] bytes = ZooKeeperProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
2106         .build().toByteArray();
2107     return ProtobufUtil.prependPBMagic(bytes);
2108   }
2109 
2110   /**
2111    * @param bytes - Content of a WAL position znode.
2112    * @return long - The current WAL position.
2113    * @throws DeserializationException
2114    */
2115   public static long parseWALPositionFrom(final byte[] bytes) throws DeserializationException {
2116     if (bytes == null) {
2117       throw new DeserializationException("Unable to parse null WAL position.");
2118     }
2119     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
2120       int pblen = ProtobufUtil.lengthOfPBMagic();
2121       ZooKeeperProtos.ReplicationHLogPosition.Builder builder =
2122           ZooKeeperProtos.ReplicationHLogPosition.newBuilder();
2123       ZooKeeperProtos.ReplicationHLogPosition position;
2124       try {
2125         ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
2126         position = builder.build();
2127       } catch (IOException e) {
2128         throw new DeserializationException(e);
2129       }
2130       return position.getPosition();
2131     } else {
2132       if (bytes.length > 0) {
2133         return Bytes.toLong(bytes);
2134       }
2135       return 0;
2136     }
2137   }
2138 
2139   /**
2140    * @param regionLastFlushedSequenceId the flushed sequence id of a region which is the min of its
2141    *          store max seq ids
2142    * @param storeSequenceIds column family to sequence Id map
2143    * @return Serialized protobuf of <code>RegionSequenceIds</code> with pb magic prefix prepended
2144    *         suitable for use to filter wal edits in distributedLogReplay mode
2145    */
2146   public static byte[] regionSequenceIdsToByteArray(final Long regionLastFlushedSequenceId,
2147       final Map<byte[], Long> storeSequenceIds) {
2148     ClusterStatusProtos.RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2149         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2150     ClusterStatusProtos.StoreSequenceId.Builder storeSequenceIdBuilder =
2151         ClusterStatusProtos.StoreSequenceId.newBuilder();
2152     if (storeSequenceIds != null) {
2153       for (Map.Entry<byte[], Long> e : storeSequenceIds.entrySet()){
2154         byte[] columnFamilyName = e.getKey();
2155         Long curSeqId = e.getValue();
2156         storeSequenceIdBuilder.setFamilyName(ByteStringer.wrap(columnFamilyName));
2157         storeSequenceIdBuilder.setSequenceId(curSeqId);
2158         regionSequenceIdsBuilder.addStoreSequenceId(storeSequenceIdBuilder.build());
2159         storeSequenceIdBuilder.clear();
2160       }
2161     }
2162     regionSequenceIdsBuilder.setLastFlushedSequenceId(regionLastFlushedSequenceId);
2163     byte[] result = regionSequenceIdsBuilder.build().toByteArray();
2164     return ProtobufUtil.prependPBMagic(result);
2165   }
2166 
2167   /**
2168    * @param bytes Content of serialized data of RegionStoreSequenceIds
2169    * @return a RegionStoreSequenceIds object
2170    * @throws DeserializationException
2171    */
2172   public static RegionStoreSequenceIds parseRegionStoreSequenceIds(final byte[] bytes)
2173       throws DeserializationException {
2174     if (bytes == null || !ProtobufUtil.isPBMagicPrefix(bytes)) {
2175       throw new DeserializationException("Unable to parse RegionStoreSequenceIds.");
2176     }
2177     RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2178         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2179     int pblen = ProtobufUtil.lengthOfPBMagic();
2180     RegionStoreSequenceIds storeIds = null;
2181     try {
2182       ProtobufUtil.mergeFrom(regionSequenceIdsBuilder, bytes, pblen, bytes.length - pblen);
2183       storeIds = regionSequenceIdsBuilder.build();
2184     } catch (IOException e) {
2185       throw new DeserializationException(e);
2186     }
2187     return storeIds;
2188   }
2189 }