View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.fail;
26  
27  import java.io.FileNotFoundException;
28  import java.io.IOException;
29  import java.util.Arrays;
30  import java.util.Comparator;
31  import java.util.Map;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.testclassification.MediumTests;
44  import org.apache.hadoop.hbase.TableDescriptors;
45  import org.apache.hadoop.hbase.TableExistsException;
46  import org.junit.Test;
47  import org.junit.experimental.categories.Category;
48  
49  
50  /**
51   * Tests for {@link FSTableDescriptors}.
52   */
53  // Do not support to be executed in he same JVM as other tests
54  @Category(MediumTests.class)
55  public class TestFSTableDescriptors {
56    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
57  
58    private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
59  
60    @Test (expected=IllegalArgumentException.class)
61    public void testRegexAgainstOldStyleTableInfo() {
62      Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
63      int i = FSTableDescriptors.getTableInfoSequenceId(p);
64      assertEquals(0, i);
65      // Assert it won't eat garbage -- that it fails
66      p = new Path("/tmp", "abc");
67      FSTableDescriptors.getTableInfoSequenceId(p);
68    }
69  
70    @Test
71    public void testCreateAndUpdate() throws IOException {
72      Path testdir = UTIL.getDataTestDir("testCreateAndUpdate");
73      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCreate"));
74      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
75      FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
76      assertTrue(fstd.createTableDescriptor(htd));
77      assertFalse(fstd.createTableDescriptor(htd));
78      FileStatus[] statuses = fs.listStatus(testdir);
79      assertTrue("statuses.length=" + statuses.length, statuses.length == 1);
80      for (int i = 0; i < 10; i++) {
81        fstd.updateTableDescriptor(htd);
82      }
83      statuses = fs.listStatus(testdir);
84      assertTrue(statuses.length == 1);
85      Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
86      statuses = fs.listStatus(tmpTableDir);
87      assertTrue(statuses.length == 0);
88    }
89  
90    @Test
91    public void testSequenceIdAdvancesOnTableInfo() throws IOException {
92      Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo");
93      HTableDescriptor htd = new HTableDescriptor(
94          TableName.valueOf("testSequenceidAdvancesOnTableInfo"));
95      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
96      FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
97      Path p0 = fstd.updateTableDescriptor(htd);
98      int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
99      Path p1 = fstd.updateTableDescriptor(htd);
100     // Assert we cleaned up the old file.
101     assertTrue(!fs.exists(p0));
102     int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
103     assertTrue(i1 == i0 + 1);
104     Path p2 = fstd.updateTableDescriptor(htd);
105     // Assert we cleaned up the old file.
106     assertTrue(!fs.exists(p1));
107     int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
108     assertTrue(i2 == i1 + 1);
109   }
110 
111   @Test
112   public void testFormatTableInfoSequenceId() {
113     Path p0 = assertWriteAndReadSequenceId(0);
114     // Assert p0 has format we expect.
115     StringBuilder sb = new StringBuilder();
116     for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
117       sb.append("0");
118     }
119     assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(), p0.getName());
120     // Check a few more.
121     Path p2 = assertWriteAndReadSequenceId(2);
122     Path p10000 = assertWriteAndReadSequenceId(10000);
123     // Get a .tablinfo that has no sequenceid suffix.
124     Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
125     FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
126     FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
127     FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
128     FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
129     Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
130     assertTrue(comparator.compare(fs, fs0) > 0);
131     assertTrue(comparator.compare(fs0, fs2) > 0);
132     assertTrue(comparator.compare(fs2, fs10000) > 0);
133   }
134 
135   private Path assertWriteAndReadSequenceId(final int i) {
136     Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
137     int ii = FSTableDescriptors.getTableInfoSequenceId(p);
138     assertEquals(i, ii);
139     return p;
140   }
141 
142   @Test
143   public void testRemoves() throws IOException {
144     final String name = "testRemoves";
145     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
146     // Cleanup old tests if any detrius laying around.
147     Path rootdir = new Path(UTIL.getDataTestDir(), name);
148     TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
149     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
150     htds.add(htd);
151     assertNotNull(htds.remove(htd.getTableName()));
152     assertNull(htds.remove(htd.getTableName()));
153   }
154 
155   @Test
156   public void testReadingHTDFromFS()
157   throws IOException {
158     final String name = "testReadingHTDFromFS";
159     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
160     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
161     Path rootdir = UTIL.getDataTestDir(name);
162     FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
163     fstd.createTableDescriptor(htd);
164     HTableDescriptor htd2 =
165       FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
166     assertTrue(htd.equals(htd2));
167   }
168 
169   @Test
170   public void testHTableDescriptors()
171   throws IOException, InterruptedException {
172     final String name = "testHTableDescriptors";
173     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
174     // Cleanup old tests if any debris laying around.
175     Path rootdir = new Path(UTIL.getDataTestDir(), name);
176     FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
177     final int count = 10;
178     // Write out table infos.
179     for (int i = 0; i < count; i++) {
180       HTableDescriptor htd = new HTableDescriptor(name + i);
181       htds.createTableDescriptor(htd);
182     }
183 
184     for (int i = 0; i < count; i++) {
185       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
186     }
187     for (int i = 0; i < count; i++) {
188       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
189     }
190     // Update the table infos
191     for (int i = 0; i < count; i++) {
192       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
193       htd.addFamily(new HColumnDescriptor("" + i));
194       htds.updateTableDescriptor(htd);
195     }
196     // Wait a while so mod time we write is for sure different.
197     Thread.sleep(100);
198     for (int i = 0; i < count; i++) {
199       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
200     }
201     for (int i = 0; i < count; i++) {
202       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
203     }
204     assertEquals(count * 4, htds.invocations);
205     assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
206                   htds.cachehits >= (count * 2));
207   }
208 
209   @Test
210   public void testHTableDescriptorsNoCache()
211   throws IOException, InterruptedException {
212     final String name = "testHTableDescriptorsNoCache";
213     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
214     // Cleanup old tests if any debris laying around.
215     Path rootdir = new Path(UTIL.getDataTestDir(), name);
216     FSTableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir,
217       false, false);
218     final int count = 10;
219     // Write out table infos.
220     for (int i = 0; i < count; i++) {
221       HTableDescriptor htd = new HTableDescriptor(name + i);
222       htds.createTableDescriptor(htd);
223     }
224 
225     for (int i = 0; i < count; i++) {
226       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
227     }
228     for (int i = 0; i < count; i++) {
229       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
230     }
231     // Update the table infos
232     for (int i = 0; i < count; i++) {
233       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
234       htd.addFamily(new HColumnDescriptor("" + i));
235       htds.updateTableDescriptor(htd);
236     }
237     // Wait a while so mod time we write is for sure different.
238     Thread.sleep(100);
239     for (int i = 0; i < count; i++) {
240       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
241     }
242     for (int i = 0; i < count; i++) {
243       assertTrue(htds.get(TableName.valueOf(name + i)) != null);
244     }
245     assertEquals(count * 4, htds.invocations);
246     assertTrue("expected=0, actual=" + htds.cachehits, htds.cachehits == 0);
247   }
248 
249   @Test
250   public void testGetAll()
251   throws IOException, InterruptedException {
252     final String name = "testGetAll";
253     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
254     // Cleanup old tests if any debris laying around.
255     Path rootdir = new Path(UTIL.getDataTestDir(), name);
256     FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
257     final int count = 4;
258     // Write out table infos.
259     for (int i = 0; i < count; i++) {
260       HTableDescriptor htd = new HTableDescriptor(name + i);
261       htds.createTableDescriptor(htd);
262     }
263     // add hbase:meta
264     HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
265     htds.createTableDescriptor(htd);
266 
267     assertTrue(htds.getAll().size() == count + 1);
268 
269   }
270 
271   @Test
272   public void testCacheConsistency()
273   throws IOException, InterruptedException {
274     final String name = "testCacheConsistency";
275     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
276     // Cleanup old tests if any debris laying around.
277     Path rootdir = new Path(UTIL.getDataTestDir(), name);
278     FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, rootdir);
279     FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs,
280       rootdir, false, false);
281 
282     final int count = 10;
283     // Write out table infos via non-cached FSTableDescriptors
284     for (int i = 0; i < count; i++) {
285       HTableDescriptor htd = new HTableDescriptor(name + i);
286       nonchtds.createTableDescriptor(htd);
287     }
288 
289     // Calls to getAll() won't increase the cache counter, do per table.
290     for (int i = 0; i < count; i++) {
291       assertTrue(chtds.get(TableName.valueOf(name + i)) != null);
292     }
293 
294     assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
295 
296     // add a new entry for hbase:meta
297     HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
298     nonchtds.createTableDescriptor(htd);
299 
300     // hbase:meta will only increase the cachehit by 1
301     assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
302 
303     for (Map.Entry entry : nonchtds.getAll().entrySet()) {
304       String t = (String) entry.getKey();
305       HTableDescriptor nchtd = (HTableDescriptor) entry.getValue();
306       assertTrue("expected " + htd.toString() +
307                      " got: " +
308                      chtds.get(TableName.valueOf(t)).toString(), (nchtd.equals(chtds.get(TableName.valueOf(t)))));
309     }
310   }
311 
312   @Test
313   public void testNoSuchTable() throws IOException {
314     final String name = "testNoSuchTable";
315     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
316     // Cleanup old tests if any detrius laying around.
317     Path rootdir = new Path(UTIL.getDataTestDir(), name);
318     TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
319     assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable")));
320   }
321 
322   @Test
323   public void testUpdates() throws IOException {
324     final String name = "testUpdates";
325     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
326     // Cleanup old tests if any detrius laying around.
327     Path rootdir = new Path(UTIL.getDataTestDir(), name);
328     TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
329     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
330     htds.add(htd);
331     htds.add(htd);
332     htds.add(htd);
333   }
334 
335   @Test
336   public void testTableInfoFileStatusComparator() {
337     FileStatus bare = new FileStatus(
338       0, false, 0, 0, -1,
339         new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
340     FileStatus future = new FileStatus(
341       0, false, 0, 0, -1,
342         new Path("/tmp/tablinfo." + System.currentTimeMillis()));
343     FileStatus farFuture = new FileStatus(
344       0, false, 0, 0, -1,
345         new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
346     FileStatus[] alist = {bare, future, farFuture};
347     FileStatus[] blist = {bare, farFuture, future};
348     FileStatus[] clist = {farFuture, bare, future};
349     Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
350     Arrays.sort(alist, c);
351     Arrays.sort(blist, c);
352     Arrays.sort(clist, c);
353     // Now assert all sorted same in way we want.
354     for (int i = 0; i < alist.length; i++) {
355       assertTrue(alist[i].equals(blist[i]));
356       assertTrue(blist[i].equals(clist[i]));
357       assertTrue(clist[i].equals(i == 0 ? farFuture : i == 1 ? future : bare));
358     }
359   }
360 
361   @Test
362   public void testReadingInvalidDirectoryFromFS() throws IOException {
363     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
364     try {
365       // .tmp dir is an invalid table name
366       new FSTableDescriptors(UTIL.getConfiguration(), fs,
367         FSUtils.getRootDir(UTIL.getConfiguration()))
368           .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
369       fail("Shouldn't be able to read a table descriptor for the archive directory.");
370     } catch (Exception e) {
371       LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
372           + e.getMessage());
373     }
374   }
375 
376   @Test
377   public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
378     Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready");
379     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(
380         "testCreateTableDescriptorUpdatesIfThereExistsAlready"));
381     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
382     FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
383     assertTrue(fstd.createTableDescriptor(htd));
384     assertFalse(fstd.createTableDescriptor(htd));
385     htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"));
386     assertTrue(fstd.createTableDescriptor(htd)); //this will re-create
387     Path tableDir = fstd.getTableDir(htd.getTableName());
388     Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
389     FileStatus[] statuses = fs.listStatus(tmpTableDir);
390     assertTrue(statuses.length == 0);
391 
392     assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
393   }
394 
395   private static class FSTableDescriptorsTest
396       extends FSTableDescriptors {
397 
398     public FSTableDescriptorsTest(FileSystem fs, Path rootdir)
399     throws IOException {
400       this(fs, rootdir, false, true);
401     }
402 
403     public FSTableDescriptorsTest(FileSystem fs, Path rootdir, boolean fsreadonly, boolean usecache)
404     throws IOException {
405       super(UTIL.getConfiguration(), fs, rootdir, fsreadonly, usecache);
406     }
407 
408     @Override
409     public HTableDescriptor get(TableName tablename)
410     throws TableExistsException, FileNotFoundException, IOException {
411       LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
412                    " HTableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
413       return super.get(tablename);
414     }
415   }
416 }
417