View Javadoc

1   /**
2    *  Copyright 2003-2006 Greg Luck
3    *
4    *  Licensed under the Apache License, Version 2.0 (the "License");
5    *  you may not use this file except in compliance with the License.
6    *  You may obtain a copy of the License at
7    *
8    *      http://www.apache.org/licenses/LICENSE-2.0
9    *
10   *  Unless required by applicable law or agreed to in writing, software
11   *  distributed under the License is distributed on an "AS IS" BASIS,
12   *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   *  See the License for the specific language governing permissions and
14   *  limitations under the License.
15   */
16  
17  package net.sf.ehcache;
18  
19  import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
20  import net.sf.ehcache.store.Store;
21  import net.sf.ehcache.store.LruMemoryStoreTest;
22  import net.sf.ehcache.distribution.JVMUtil;
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.Random;
30  
31  /**
32   * Other than policy differences, the Store implementations should work identically
33   *
34   * @author Greg Luck
35   * @version $Id: MemoryStoreTester.java 53 2006-04-25 08:56:21Z gregluck $
36   */
37  public class MemoryStoreTester extends AbstractCacheTest {
38  
39      private static final Log LOG = LogFactory.getLog(MemoryStoreTester.class.getName());
40  
41      /**
42       * The memory store that tests will be performed on
43       */
44      protected Store store;
45  
46      /**
47       * For automatic suite generators
48       */
49      public void testNoop() {
50          //noop
51      }
52  
53      /**
54       * setup test
55       */
56      protected void setUp() throws Exception {
57          manager = CacheManager.getInstance();
58      }
59  
60      /**
61       * teardown
62       */
63      protected void tearDown() throws Exception {
64          if (manager != null) {
65              manager.shutdown();
66          }
67      }
68  
69      /**
70       * Creates a cache with the given policy and adds it to the manager.
71       *
72       * @param evictionPolicy
73       * @throws CacheException
74       */
75      protected void createMemoryStore(MemoryStoreEvictionPolicy evictionPolicy) throws CacheException {
76          Cache cache = new Cache("test", 1000, evictionPolicy, true, null, true, 60, 30, false, 60, null);
77          manager.addCache(cache);
78          store = cache.getMemoryStore();
79      }
80  
81      /**
82       * Creates a cache with the given policy and adds it to the manager.
83       *
84       * @param evictionPolicy
85       * @throws CacheException
86       */
87      protected void createMemoryStore(MemoryStoreEvictionPolicy evictionPolicy, int memoryStoreSize) throws CacheException {
88          manager.removeCache("test");
89          Cache cache = new Cache("test", memoryStoreSize, evictionPolicy, true, null, true, 60, 30, false, 60, null);
90          manager.addCache(cache);
91          store = cache.getMemoryStore();
92      }
93  
94      /**
95       * Creates a store from the given configuration and cache within it.
96       *
97       * @param filePath
98       * @param cacheName
99       * @throws CacheException
100      */
101     protected void createMemoryStore(String filePath, String cacheName) throws CacheException {
102         manager.shutdown();
103         manager = CacheManager.create(filePath);
104         Cache cache = manager.getCache(cacheName);
105         store = cache.getMemoryStore();
106     }
107 
108 
109     /**
110      * Test elements can be put in the store
111      */
112     protected void putTest() throws IOException {
113         Element element;
114 
115         assertEquals(0, store.getSize());
116 
117         element = new Element("key1", "value1");
118         store.put(element);
119         assertEquals(1, store.getSize());
120         assertEquals("value1", store.get("key1").getObjectValue());
121 
122         element = new Element("key2", "value2");
123         store.put(element);
124         assertEquals(2, store.getSize());
125         assertEquals("value2", store.get("key2").getObjectValue());
126     }
127 
128     /**
129      * Test elements can be removed from the store
130      */
131     protected void removeTest() throws IOException {
132         Element element;
133 
134         element = new Element("key1", "value1");
135         store.put(element);
136         assertEquals(1, store.getSize());
137 
138         store.remove("key1");
139         assertEquals(0, store.getSize());
140 
141         store.put(new Element("key2", "value2"));
142         store.put(new Element("key3", "value3"));
143         assertEquals(2, store.getSize());
144 
145         assertNotNull(store.remove("key2"));
146         assertEquals(1, store.getSize());
147 
148         // Try to remove an object that is not there in the store
149         assertNull(store.remove("key4"));
150         assertEquals(1, store.getSize());
151 
152         //check no NPE on key handling
153         assertNull(store.remove(null));
154     }
155 
156 
157     /**
158      * Check no NPE on put
159      */
160     public void testNullPut() throws IOException {
161         store.put(null);
162     }
163 
164     /**
165      * Check no NPE on get
166      */
167     public void testNullGet() throws IOException {
168         assertNull(store.get(null));
169     }
170 
171     /**
172      * Check no NPE on remove
173      */
174     public void testNullRemove() throws IOException {
175         assertNull(store.remove(null));
176     }
177 
178     /**
179      * Tests looking up an entry that does not exist.
180      */
181     public void testGetUnknown() throws Exception {
182         final Element element = store.get("key");
183         assertNull(element);
184     }
185 
186     /**
187      * Tests adding an entry.
188      */
189     public void testPut() throws Exception {
190         final String value = "value";
191         final String key = "key";
192 
193         // Make sure the element is not found
194         assertEquals(0, store.getSize());
195         Element element = store.get(key);
196         assertNull(element);
197 
198         // Add the element
199         element = new Element(key, value);
200         store.put(element);
201 
202         // Get the element
203         assertEquals(1, store.getSize());
204         element = store.get(key);
205         assertNotNull(element);
206         assertEquals(value, element.getObjectValue());
207     }
208 
209     /**
210      * Tests removing an entry.
211      */
212     public void testRemove() throws Exception {
213         final String value = "value";
214         final String key = "key";
215 
216         // Add the entry
217 
218         Element element = new Element(key, value);
219         store.put(element);
220 
221         // Check the entry is there
222         assertEquals(1, store.getSize());
223         element = store.get(key);
224         assertNotNull(element);
225 
226         // Remove it
227         store.remove(key);
228 
229         // Check the entry is not there
230         assertEquals(0, store.getSize());
231         element = store.get(key);
232         assertNull(element);
233     }
234 
235     /**
236      * Tests removing all the entries.
237      */
238     public void testRemoveAll() throws Exception {
239         final String value = "value";
240         final String key = "key";
241 
242         // Add the entry
243         Element element = new Element(key, value);
244         store.put(element);
245 
246         // Check the entry is there
247         element = store.get(key);
248         assertNotNull(element);
249 
250         // Remove it
251         store.removeAll();
252 
253         // Check the entry is not there
254         assertEquals(0, store.getSize());
255         element = store.get(key);
256         assertNull(element);
257     }
258 
259     /**
260      * Tests bulk load.
261      */
262     public void testBulkLoad() throws Exception {
263         final Random random = new Random();
264         StopWatch stopWatch = new StopWatch();
265 
266 
267         // Add a bunch of entries
268         for (int i = 0; i < 500; i++) {
269             // Use a random length value
270             final String key = "key" + i;
271             final String value = "value" + random.nextInt(1000);
272 
273             // Add an element, and make sure it is present
274             Element element = new Element(key, value);
275             store.put(element);
276             element = store.get(key);
277             assertNotNull(element);
278 
279             // Remove the element
280             store.remove(key);
281             element = store.get(key);
282             assertNull(element);
283 
284             element = new Element(key, value);
285             store.put(element);
286             element = store.get(key);
287             assertNotNull(element);
288         }
289         long time = stopWatch.getElapsedTime();
290         LOG.info("Time for Bulk Load: " + time);
291     }
292 
293     /**
294      * Benchmark to test speed.
295      */
296     public void testBenchmarkPutGetRemove() throws Exception {
297         final String key = "key";
298         byte[] value = new byte[500];
299         StopWatch stopWatch = new StopWatch();
300 
301         // Add a bunch of entries
302         for (int i = 0; i < 50000; i++) {
303             Element element = new Element(key, value);
304             store.put(element);
305             store.get(key + i);
306         }
307         for (int i = 0; i < 50000; i++) {
308             store.remove(key + i);
309         }
310         long time = stopWatch.getElapsedTime();
311         LOG.info("Time for benchmarkPutGetRemove: " + time);
312         assertTrue("Too slow. Time was " + time, time < 500);
313     }
314 
315     /**
316      * Benchmark to test speed.
317      */
318     public void testBenchmarkPutGet() throws Exception {
319         final String key = "key";
320         byte[] value = new byte[500];
321         StopWatch stopWatch = new StopWatch();
322 
323         // Add a bunch of entries
324         for (int i = 0; i < 50000; i++) {
325             Element element = new Element(key, value);
326             store.put(element);
327         }
328         for (int i = 0; i < 50000; i++) {
329             store.get(key + i);
330         }
331         long time = stopWatch.getElapsedTime();
332         LOG.info("Time for benchmarkPutGet: " + time);
333         assertTrue("Too slow. Time was " + time, time < 300);
334     }
335 
336 
337     /**
338      * Benchmark to test speed.
339      * Original implementation 12seconds
340      * This implementation 9 seconds
341      */
342     public void benchmarkPutGetSuryaTest(long allowedTime) throws Exception {
343         Random random = new Random();
344         byte[] value = new byte[500];
345         StopWatch stopWatch = new StopWatch();
346 
347         // Add a bunch of entries
348         for (int i = 0; i < 50000; i++) {
349             String key = "key" + i;
350 
351             Element element = new Element(key, value);
352             store.put(element);
353 
354             //Access each element random number of times, min:0 maximum:9
355             int accesses = random.nextInt(5);
356             for (int j = 0; j <= accesses; j++) {
357                 store.get(key);
358             }
359         }
360         long time = stopWatch.getElapsedTime();
361         LOG.info("Time for benchmarkPutGetSurya: " + time);
362         assertTrue("Too slow. Time was " + time, time < allowedTime);
363     }
364 
365     /**
366      * Multi-thread read-only test.
367      */
368     public void testReadOnlyThreads() throws Exception {
369 
370         // Add a couple of elements
371         store.put(new Element("key0", "value"));
372         store.put(new Element("key1", "value"));
373 
374         // Run a set of threads, that attempt to fetch the elements
375         final List executables = new ArrayList();
376         for (int i = 0; i < 10; i++) {
377             final String key = "key" + (i % 2);
378             final MemoryStoreTester.Executable executable = new LruMemoryStoreTest.Executable() {
379                 public void execute() throws Exception {
380                     final Element element = store.get(key);
381                     assertNotNull(element);
382                     assertEquals("value", element.getObjectValue());
383                 }
384             };
385             executables.add(executable);
386         }
387         runThreads(executables);
388     }
389 
390     /**
391      * Multi-thread read-write test.
392      */
393     public void testReadWriteThreads() throws Exception {
394 
395         final String value = "value";
396         final String key = "key";
397 
398         // Add the entry
399         final Element element = new Element(key, value);
400         store.put(element);
401 
402         // Run a set of threads that get, put and remove an entry
403         final List executables = new ArrayList();
404         for (int i = 0; i < 5; i++) {
405             final MemoryStoreTester.Executable executable = new MemoryStoreTester.Executable() {
406                 public void execute() throws Exception {
407                     final Element element = store.get("key");
408                     assertNotNull(element);
409                 }
410             };
411             executables.add(executable);
412         }
413         for (int i = 0; i < 5; i++) {
414             final MemoryStoreTester.Executable executable = new MemoryStoreTester.Executable() {
415                 public void execute() throws Exception {
416                     store.put(element);
417                 }
418             };
419             executables.add(executable);
420         }
421 
422         runThreads(executables);
423     }
424 
425     /**
426      * Multi-thread read, put and removeAll test.
427      * This checks for memory leaks
428      * using the removeAll which was the known cause of memory leaks with LruMemoryStore in JCS
429      */
430     public void testMemoryLeak() throws Exception {
431         long differenceMemoryCache = thrashCache();
432         assertTrue(differenceMemoryCache < 500000);
433     }
434 
435 
436     /**
437      * This method tries to get the store too leak.
438      */
439     protected long thrashCache() throws Exception {
440 
441 
442         long startingSize = measureMemoryUse();
443         LOG.info("Memory Used is: " + startingSize);
444 
445         final String value = "value";
446         final String key = "key";
447 
448         // Add the entry
449         final Element element = new Element(key, value);
450         store.put(element);
451 
452         // Create 15 threads that read the keys;
453         final List executables = new ArrayList();
454         for (int i = 0; i < 15; i++) {
455             final LruMemoryStoreTest.Executable executable = new MemoryStoreTester.Executable() {
456                 public void execute() throws Exception {
457                     for (int i = 0; i < 500; i++) {
458                         final String key = "key" + i;
459                         store.get(key);
460                     }
461                     store.get("key");
462                 }
463             };
464             executables.add(executable);
465         }
466         //Create 15 threads that are insert 500 keys with large byte[] as values
467         for (int i = 0; i < 15; i++) {
468             final LruMemoryStoreTest.Executable executable = new MemoryStoreTester.Executable() {
469                 public void execute() throws Exception {
470 
471                     // Add a bunch of entries
472                     for (int i = 0; i < 500; i++) {
473                         // Use a random length value
474                         final String key = "key" + i;
475                         byte[] value = new byte[10000];
476                         Element element = new Element(key, value);
477                         store.put(element);
478                     }
479                 }
480             };
481             executables.add(executable);
482         }
483 
484         runThreads(executables);
485         store.removeAll();
486 
487         long finishingSize = measureMemoryUse();
488         LOG.info("Memory Used is: " + finishingSize);
489         return finishingSize - startingSize;
490     }
491 
492 
493     /**
494      * Multi-thread read-write test.
495      */
496     public void testReadWriteThreadsSurya() throws Exception {
497 
498         long start = System.currentTimeMillis();
499         final List executables = new ArrayList();
500         final Random random = new Random();
501 
502         // 50% of the time get data
503         for (int i = 0; i < 10; i++) {
504             final Executable executable = new Executable() {
505                 public void execute() throws Exception {
506                     store.get("key" + random.nextInt(10000));
507                 }
508             };
509             executables.add(executable);
510         }
511 
512         //25% of the time add data
513         for (int i = 0; i < 5; i++) {
514             final Executable executable = new Executable() {
515                 public void execute() throws Exception {
516                     store.put(new Element("key" + random.nextInt(20000), "value"));
517                 }
518             };
519             executables.add(executable);
520         }
521 
522         //25% if the time remove the data
523         for (int i = 0; i < 5; i++) {
524             final Executable executable = new Executable() {
525                 public void execute() throws Exception {
526                     store.remove("key" + random.nextInt(10000));
527                 }
528             };
529             executables.add(executable);
530         }
531 
532         runThreads(executables);
533         long end = System.currentTimeMillis();
534         LOG.info("Total time for the test: " + (end + start) + " ms");
535     }
536 
537 
538     /**
539      * Runs a set of threads, for a fixed amount of time.
540      */
541     protected void runThreads(final List executables) throws Exception {
542 
543         final long endTime = System.currentTimeMillis() + 10000;
544         final Throwable[] errors = new Throwable[1];
545 
546         // Spin up the threads
547         final Thread[] threads = new Thread[executables.size()];
548         for (int i = 0; i < threads.length; i++) {
549             final LruMemoryStoreTest.Executable executable = (MemoryStoreTester.Executable) executables.get(i);
550             threads[i] = new Thread() {
551                 public void run() {
552                     try {
553                         // Run the thread until the given end time
554                         while (System.currentTimeMillis() < endTime) {
555                             executable.execute();
556                         }
557                     } catch (Throwable t) {
558                         // Hang on to any errors
559                         errors[0] = t;
560                     }
561                 }
562             };
563             threads[i].start();
564         }
565 
566         // Wait for the threads to finish
567         for (int i = 0; i < threads.length; i++) {
568             threads[i].join();
569         }
570 
571         // Throw any error that happened
572         if (errors[0] != null) {
573             throw new Exception("Test thread failed.", errors[0]);
574         }
575     }
576 
577 
578     /**
579      * A runnable, that can throw an exception.
580      */
581     protected interface Executable {
582         /**
583          * Executes this object.
584          *
585          * @throws Exception
586          */
587         void execute() throws Exception;
588     }
589 
590 
591 
592     /**
593      * Test behaviour of memory store using 1 million records.
594      * This is expected to run out of memory on a 64MB machine. Where it runs out
595      * is asserted so that design changes do not start using more memory per element.
596      * <p/>
597      * This test will fail (ie not get an out of memory error) on VMs configured to be server which do not have a fixed upper memory limit.
598      * <p/>
599      * Takes too long to run therefore switch off
600      * <p/>
601      * These memory size asserts were 100,000 and 60,000. The ApacheLRU map does not get quite as high numbers. 
602      */
603     public void testMemoryStoreOutOfMemoryLimit() throws Exception {
604         //Set size so the second element overflows to disk.
605         Cache cache = new Cache("memoryLimitTest", 1000000, false, false, 500, 500);
606         manager.addCache(cache);
607         int i = 0;
608         try {
609             for (; i < 1000000; i++) {
610                 cache.put(new Element("" +
611                         i, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
612                         + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
613                         + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
614                         + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
615                         + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
616                         + "AAAAA " + i));
617             }
618             fail();
619         } catch (OutOfMemoryError e) {
620             if (JVMUtil.isJDK15()) {
621                 assertTrue(i > 90000);
622             } else {
623                 assertTrue(i > 50000);
624             }
625             LOG.info("Ran out of memory putting " + i + "th element");
626         }
627     }
628 
629 }