1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.util.Arrays;
30 import java.util.Comparator;
31 import java.util.Map;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.testclassification.MediumTests;
44 import org.apache.hadoop.hbase.TableDescriptors;
45 import org.apache.hadoop.hbase.TableExistsException;
46 import org.junit.Test;
47 import org.junit.experimental.categories.Category;
48
49
50
51
52
53
54 @Category(MediumTests.class)
55 public class TestFSTableDescriptors {
56 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
57
58 private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
59
60 @Test (expected=IllegalArgumentException.class)
61 public void testRegexAgainstOldStyleTableInfo() {
62 Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
63 int i = FSTableDescriptors.getTableInfoSequenceId(p);
64 assertEquals(0, i);
65
66 p = new Path("/tmp", "abc");
67 FSTableDescriptors.getTableInfoSequenceId(p);
68 }
69
70 @Test
71 public void testCreateAndUpdate() throws IOException {
72 Path testdir = UTIL.getDataTestDir("testCreateAndUpdate");
73 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCreate"));
74 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
75 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
76 assertTrue(fstd.createTableDescriptor(htd));
77 assertFalse(fstd.createTableDescriptor(htd));
78 FileStatus[] statuses = fs.listStatus(testdir);
79 assertTrue("statuses.length=" + statuses.length, statuses.length == 1);
80 for (int i = 0; i < 10; i++) {
81 fstd.updateTableDescriptor(htd);
82 }
83 statuses = fs.listStatus(testdir);
84 assertTrue(statuses.length == 1);
85 Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
86 statuses = fs.listStatus(tmpTableDir);
87 assertTrue(statuses.length == 0);
88 }
89
90 @Test
91 public void testSequenceIdAdvancesOnTableInfo() throws IOException {
92 Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo");
93 HTableDescriptor htd = new HTableDescriptor(
94 TableName.valueOf("testSequenceidAdvancesOnTableInfo"));
95 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
96 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
97 Path p0 = fstd.updateTableDescriptor(htd);
98 int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
99 Path p1 = fstd.updateTableDescriptor(htd);
100
101 assertTrue(!fs.exists(p0));
102 int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
103 assertTrue(i1 == i0 + 1);
104 Path p2 = fstd.updateTableDescriptor(htd);
105
106 assertTrue(!fs.exists(p1));
107 int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
108 assertTrue(i2 == i1 + 1);
109 }
110
111 @Test
112 public void testFormatTableInfoSequenceId() {
113 Path p0 = assertWriteAndReadSequenceId(0);
114
115 StringBuilder sb = new StringBuilder();
116 for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
117 sb.append("0");
118 }
119 assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(), p0.getName());
120
121 Path p2 = assertWriteAndReadSequenceId(2);
122 Path p10000 = assertWriteAndReadSequenceId(10000);
123
124 Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
125 FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
126 FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
127 FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
128 FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
129 Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
130 assertTrue(comparator.compare(fs, fs0) > 0);
131 assertTrue(comparator.compare(fs0, fs2) > 0);
132 assertTrue(comparator.compare(fs2, fs10000) > 0);
133 }
134
135 private Path assertWriteAndReadSequenceId(final int i) {
136 Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
137 int ii = FSTableDescriptors.getTableInfoSequenceId(p);
138 assertEquals(i, ii);
139 return p;
140 }
141
142 @Test
143 public void testRemoves() throws IOException {
144 final String name = "testRemoves";
145 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
146
147 Path rootdir = new Path(UTIL.getDataTestDir(), name);
148 TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
149 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
150 htds.add(htd);
151 assertNotNull(htds.remove(htd.getTableName()));
152 assertNull(htds.remove(htd.getTableName()));
153 }
154
155 @Test
156 public void testReadingHTDFromFS()
157 throws IOException {
158 final String name = "testReadingHTDFromFS";
159 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
160 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
161 Path rootdir = UTIL.getDataTestDir(name);
162 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
163 fstd.createTableDescriptor(htd);
164 HTableDescriptor htd2 =
165 FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
166 assertTrue(htd.equals(htd2));
167 }
168
169 @Test
170 public void testHTableDescriptors()
171 throws IOException, InterruptedException {
172 final String name = "testHTableDescriptors";
173 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
174
175 Path rootdir = new Path(UTIL.getDataTestDir(), name);
176 FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
177 final int count = 10;
178
179 for (int i = 0; i < count; i++) {
180 HTableDescriptor htd = new HTableDescriptor(name + i);
181 htds.createTableDescriptor(htd);
182 }
183
184 for (int i = 0; i < count; i++) {
185 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
186 }
187 for (int i = 0; i < count; i++) {
188 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
189 }
190
191 for (int i = 0; i < count; i++) {
192 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
193 htd.addFamily(new HColumnDescriptor("" + i));
194 htds.updateTableDescriptor(htd);
195 }
196
197 Thread.sleep(100);
198 for (int i = 0; i < count; i++) {
199 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
200 }
201 for (int i = 0; i < count; i++) {
202 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
203 }
204 assertEquals(count * 4, htds.invocations);
205 assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
206 htds.cachehits >= (count * 2));
207 }
208
209 @Test
210 public void testHTableDescriptorsNoCache()
211 throws IOException, InterruptedException {
212 final String name = "testHTableDescriptorsNoCache";
213 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
214
215 Path rootdir = new Path(UTIL.getDataTestDir(), name);
216 FSTableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir,
217 false, false);
218 final int count = 10;
219
220 for (int i = 0; i < count; i++) {
221 HTableDescriptor htd = new HTableDescriptor(name + i);
222 htds.createTableDescriptor(htd);
223 }
224
225 for (int i = 0; i < count; i++) {
226 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
227 }
228 for (int i = 0; i < count; i++) {
229 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
230 }
231
232 for (int i = 0; i < count; i++) {
233 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
234 htd.addFamily(new HColumnDescriptor("" + i));
235 htds.updateTableDescriptor(htd);
236 }
237
238 Thread.sleep(100);
239 for (int i = 0; i < count; i++) {
240 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
241 }
242 for (int i = 0; i < count; i++) {
243 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
244 }
245 assertEquals(count * 4, htds.invocations);
246 assertTrue("expected=0, actual=" + htds.cachehits, htds.cachehits == 0);
247 }
248
249 @Test
250 public void testGetAll()
251 throws IOException, InterruptedException {
252 final String name = "testGetAll";
253 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
254
255 Path rootdir = new Path(UTIL.getDataTestDir(), name);
256 FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
257 final int count = 4;
258
259 for (int i = 0; i < count; i++) {
260 HTableDescriptor htd = new HTableDescriptor(name + i);
261 htds.createTableDescriptor(htd);
262 }
263
264 HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
265 htds.createTableDescriptor(htd);
266
267 assertTrue(htds.getAll().size() == count + 1);
268
269 }
270
271 @Test
272 public void testCacheConsistency()
273 throws IOException, InterruptedException {
274 final String name = "testCacheConsistency";
275 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
276
277 Path rootdir = new Path(UTIL.getDataTestDir(), name);
278 FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, rootdir);
279 FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs,
280 rootdir, false, false);
281
282 final int count = 10;
283
284 for (int i = 0; i < count; i++) {
285 HTableDescriptor htd = new HTableDescriptor(name + i);
286 nonchtds.createTableDescriptor(htd);
287 }
288
289
290 for (int i = 0; i < count; i++) {
291 assertTrue(chtds.get(TableName.valueOf(name + i)) != null);
292 }
293
294 assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
295
296
297 HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
298 nonchtds.createTableDescriptor(htd);
299
300
301 assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
302
303 for (Map.Entry entry : nonchtds.getAll().entrySet()) {
304 String t = (String) entry.getKey();
305 HTableDescriptor nchtd = (HTableDescriptor) entry.getValue();
306 assertTrue("expected " + htd.toString() +
307 " got: " +
308 chtds.get(TableName.valueOf(t)).toString(), (nchtd.equals(chtds.get(TableName.valueOf(t)))));
309 }
310 }
311
312 @Test
313 public void testNoSuchTable() throws IOException {
314 final String name = "testNoSuchTable";
315 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
316
317 Path rootdir = new Path(UTIL.getDataTestDir(), name);
318 TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
319 assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable")));
320 }
321
322 @Test
323 public void testUpdates() throws IOException {
324 final String name = "testUpdates";
325 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
326
327 Path rootdir = new Path(UTIL.getDataTestDir(), name);
328 TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
329 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
330 htds.add(htd);
331 htds.add(htd);
332 htds.add(htd);
333 }
334
335 @Test
336 public void testTableInfoFileStatusComparator() {
337 FileStatus bare = new FileStatus(
338 0, false, 0, 0, -1,
339 new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
340 FileStatus future = new FileStatus(
341 0, false, 0, 0, -1,
342 new Path("/tmp/tablinfo." + System.currentTimeMillis()));
343 FileStatus farFuture = new FileStatus(
344 0, false, 0, 0, -1,
345 new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
346 FileStatus[] alist = {bare, future, farFuture};
347 FileStatus[] blist = {bare, farFuture, future};
348 FileStatus[] clist = {farFuture, bare, future};
349 Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
350 Arrays.sort(alist, c);
351 Arrays.sort(blist, c);
352 Arrays.sort(clist, c);
353
354 for (int i = 0; i < alist.length; i++) {
355 assertTrue(alist[i].equals(blist[i]));
356 assertTrue(blist[i].equals(clist[i]));
357 assertTrue(clist[i].equals(i == 0 ? farFuture : i == 1 ? future : bare));
358 }
359 }
360
361 @Test
362 public void testReadingInvalidDirectoryFromFS() throws IOException {
363 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
364 try {
365
366 new FSTableDescriptors(UTIL.getConfiguration(), fs,
367 FSUtils.getRootDir(UTIL.getConfiguration()))
368 .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
369 fail("Shouldn't be able to read a table descriptor for the archive directory.");
370 } catch (Exception e) {
371 LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
372 + e.getMessage());
373 }
374 }
375
376 @Test
377 public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
378 Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready");
379 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(
380 "testCreateTableDescriptorUpdatesIfThereExistsAlready"));
381 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
382 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
383 assertTrue(fstd.createTableDescriptor(htd));
384 assertFalse(fstd.createTableDescriptor(htd));
385 htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"));
386 assertTrue(fstd.createTableDescriptor(htd));
387 Path tableDir = fstd.getTableDir(htd.getTableName());
388 Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
389 FileStatus[] statuses = fs.listStatus(tmpTableDir);
390 assertTrue(statuses.length == 0);
391
392 assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
393 }
394
395 private static class FSTableDescriptorsTest
396 extends FSTableDescriptors {
397
398 public FSTableDescriptorsTest(FileSystem fs, Path rootdir)
399 throws IOException {
400 this(fs, rootdir, false, true);
401 }
402
403 public FSTableDescriptorsTest(FileSystem fs, Path rootdir, boolean fsreadonly, boolean usecache)
404 throws IOException {
405 super(UTIL.getConfiguration(), fs, rootdir, fsreadonly, usecache);
406 }
407
408 @Override
409 public HTableDescriptor get(TableName tablename)
410 throws TableExistsException, FileNotFoundException, IOException {
411 LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
412 " HTableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
413 return super.get(tablename);
414 }
415 }
416 }
417