View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.replication;
20  
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.TreeMap;
25  
26  import org.apache.hadoop.hbase.Cell;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.HRegionInfo;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.testclassification.SmallTests;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.wal.WAL.Entry;
33  import org.apache.hadoop.hbase.wal.WALKey;
34  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.junit.Assert;
37  import org.junit.Test;
38  import org.junit.experimental.categories.Category;
39  
40  import com.google.common.collect.Lists;
41  
42  import static org.junit.Assert.*;
43  import static org.mockito.Mockito.*;
44  
45  @Category(SmallTests.class)
46  public class TestReplicationWALEntryFilters {
47  
48    static byte[] a = new byte[] {'a'};
49    static byte[] b = new byte[] {'b'};
50    static byte[] c = new byte[] {'c'};
51    static byte[] d = new byte[] {'d'};
52  
53    @Test
54    public void testSystemTableWALEntryFilter() {
55      SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter();
56  
57      // meta
58      WALKey key1 = new WALKey( HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
59        TableName.META_TABLE_NAME);
60      Entry metaEntry = new Entry(key1, null);
61  
62      assertNull(filter.filter(metaEntry));
63  
64      // ns table
65      WALKey key2 = new WALKey(new byte[] {}, TableName.NAMESPACE_TABLE_NAME);
66      Entry nsEntry = new Entry(key2, null);
67      assertNull(filter.filter(nsEntry));
68  
69      // user table
70  
71      WALKey key3 = new WALKey(new byte[] {}, TableName.valueOf("foo"));
72      Entry userEntry = new Entry(key3, null);
73  
74      assertEquals(userEntry, filter.filter(userEntry));
75    }
76  
77    @Test
78    public void testScopeWALEntryFilter() {
79      ScopeWALEntryFilter filter = new ScopeWALEntryFilter();
80  
81      Entry userEntry = createEntry(a, b);
82      Entry userEntryA = createEntry(a);
83      Entry userEntryB = createEntry(b);
84      Entry userEntryEmpty = createEntry();
85  
86      // no scopes
87      assertEquals(null, filter.filter(userEntry));
88  
89      // empty scopes
90      TreeMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
91      userEntry = createEntry(a, b);
92      userEntry.getKey().setScopes(scopes);
93      assertEquals(null, filter.filter(userEntry));
94  
95      // different scope
96      scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
97      scopes.put(c, HConstants.REPLICATION_SCOPE_GLOBAL);
98      userEntry = createEntry(a, b);
99      userEntry.getKey().setScopes(scopes);
100     // all kvs should be filtered
101     assertEquals(userEntryEmpty, filter.filter(userEntry));
102 
103     // local scope
104     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
105     scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
106     userEntry = createEntry(a, b);
107     userEntry.getKey().setScopes(scopes);
108     assertEquals(userEntryEmpty, filter.filter(userEntry));
109     scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
110     assertEquals(userEntryEmpty, filter.filter(userEntry));
111 
112     // only scope a
113     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
114     scopes.put(a, HConstants.REPLICATION_SCOPE_GLOBAL);
115     userEntry = createEntry(a, b);
116     userEntry.getKey().setScopes(scopes);
117     assertEquals(userEntryA, filter.filter(userEntry));
118     scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
119     assertEquals(userEntryA, filter.filter(userEntry));
120 
121     // only scope b
122     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
123     scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
124     userEntry = createEntry(a, b);
125     userEntry.getKey().setScopes(scopes);
126     assertEquals(userEntryB, filter.filter(userEntry));
127     scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
128     assertEquals(userEntryB, filter.filter(userEntry));
129 
130     // scope a and b
131     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
132     scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
133     userEntry = createEntry(a, b);
134     userEntry.getKey().setScopes(scopes);
135     assertEquals(userEntryB, filter.filter(userEntry));
136     scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
137     assertEquals(userEntryB, filter.filter(userEntry));
138   }
139 
140   WALEntryFilter nullFilter = new WALEntryFilter() {
141     @Override
142     public Entry filter(Entry entry) {
143       return null;
144     }
145   };
146 
147   WALEntryFilter passFilter = new WALEntryFilter() {
148     @Override
149     public Entry filter(Entry entry) {
150       return entry;
151     }
152   };
153 
154   @Test
155   public void testChainWALEntryFilter() {
156     Entry userEntry = createEntry(a, b, c);
157 
158     ChainWALEntryFilter filter = new ChainWALEntryFilter(passFilter);
159     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
160 
161     filter = new ChainWALEntryFilter(passFilter, passFilter);
162     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
163 
164     filter = new ChainWALEntryFilter(passFilter, passFilter, passFilter);
165     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
166 
167     filter = new ChainWALEntryFilter(nullFilter);
168     assertEquals(null, filter.filter(userEntry));
169 
170     filter = new ChainWALEntryFilter(nullFilter, passFilter);
171     assertEquals(null, filter.filter(userEntry));
172 
173     filter = new ChainWALEntryFilter(passFilter, nullFilter);
174     assertEquals(null, filter.filter(userEntry));
175 
176     filter = new ChainWALEntryFilter(nullFilter, passFilter, nullFilter);
177     assertEquals(null, filter.filter(userEntry));
178 
179     filter = new ChainWALEntryFilter(nullFilter, nullFilter);
180     assertEquals(null, filter.filter(userEntry));
181 
182     // flatten
183     filter =
184         new ChainWALEntryFilter(
185           new ChainWALEntryFilter(passFilter,
186             new ChainWALEntryFilter(passFilter, passFilter),
187           new ChainWALEntryFilter(passFilter),
188           new ChainWALEntryFilter(passFilter)),
189           new ChainWALEntryFilter(passFilter));
190     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
191 
192 
193     filter =
194         new ChainWALEntryFilter(
195           new ChainWALEntryFilter(passFilter,
196             new ChainWALEntryFilter(passFilter,
197               new ChainWALEntryFilter(nullFilter))),
198           new ChainWALEntryFilter(passFilter));
199     assertEquals(null, filter.filter(userEntry));
200   }
201 
202   @Test
203   public void testTableCfWALEntryFilter() {
204     ReplicationPeer peer = mock(ReplicationPeer.class);
205 
206     when(peer.getTableCFs()).thenReturn(null);
207     Entry userEntry = createEntry(a, b, c);
208     TableCfWALEntryFilter filter = new TableCfWALEntryFilter(peer);
209     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
210 
211     // empty map
212     userEntry = createEntry(a, b, c);
213     Map<TableName, List<String>> tableCfs = new HashMap<TableName, List<String>>();
214     when(peer.getTableCFs()).thenReturn(tableCfs);
215     filter = new TableCfWALEntryFilter(peer);
216     assertEquals(null, filter.filter(userEntry));
217 
218     // table bar
219     userEntry = createEntry(a, b, c);
220     tableCfs = new HashMap<TableName, List<String>>();
221     tableCfs.put(TableName.valueOf("bar"), null);
222     when(peer.getTableCFs()).thenReturn(tableCfs);
223     filter = new TableCfWALEntryFilter(peer);
224     assertEquals(null, filter.filter(userEntry));
225 
226     // table foo:a
227     userEntry = createEntry(a, b, c);
228     tableCfs = new HashMap<TableName, List<String>>();
229     tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a"));
230     when(peer.getTableCFs()).thenReturn(tableCfs);
231     filter = new TableCfWALEntryFilter(peer);
232     assertEquals(createEntry(a), filter.filter(userEntry));
233 
234     // table foo:a,c
235     userEntry = createEntry(a, b, c, d);
236     tableCfs = new HashMap<TableName, List<String>>();
237     tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a", "c"));
238     when(peer.getTableCFs()).thenReturn(tableCfs);
239     filter = new TableCfWALEntryFilter(peer);
240     assertEquals(createEntry(a,c), filter.filter(userEntry));
241   }
242 
243   private Entry createEntry(byte[]... kvs) {
244     WALKey key1 = new WALKey(new byte[] {}, TableName.valueOf("foo"));
245     WALEdit edit1 = new WALEdit();
246 
247     for (byte[] kv : kvs) {
248       edit1.add(new KeyValue(kv, kv, kv));
249     }
250     return new Entry(key1, edit1);
251   }
252 
253 
254   private void assertEquals(Entry e1, Entry e2) {
255     Assert.assertEquals(e1 == null, e2 == null);
256     if (e1 == null) {
257       return;
258     }
259 
260     // do not compare WALKeys
261 
262     // compare kvs
263     Assert.assertEquals(e1.getEdit() == null, e2.getEdit() == null);
264     if (e1.getEdit() == null) {
265       return;
266     }
267     List<Cell> cells1 = e1.getEdit().getCells();
268     List<Cell> cells2 = e2.getEdit().getCells();
269     Assert.assertEquals(cells1.size(), cells2.size());
270     for (int i = 0; i < cells1.size(); i++) {
271       KeyValue.COMPARATOR.compare(cells1.get(i), cells2.get(i));
272     }
273   }
274 }