Overflowing Data to Disk Example

Error formatting macro: composition-setup: java.lang.NoClassDefFoundError: net/customware/confluence/plugin/composition/CompositionUtil

You can use disk with your in-memory caching to overflow data that is too large for your memory. Overflow stores some data on disk while your region is larger than a configured size. Overflow management is transparent to your application; the disk files are treated as an extension of the region in memory. Disk data is maintained just like in-memory data, and entry requests are automatically retrieved from the disk files as needed. This is a great alternative to eviction for applications with both a large volume of data and a high cost associated with accessing the original, outside data source. Read more about overflow.

Note: Overflow does not provide a backup of your data; it provides a disk extension to your in-memory storage space for the region. If you want to back up your data, see the Persisting Data to Disk Example.

Running the Example

In this example overflow data is written to the subdirectory overflowData1. Notice that the disk files are only there while the region is in memory. When the region is removed from memory, its overflow files are no longer needed, so they are destroyed.

To run this example, you must have terminal sessions configured for the QuickStart examples, as described in Setting Up the Environment.

In a single session, run the example:

$ java quickstart.DataOverflow

Example Listings

Program and cache configuration file listings:

Error formatting macro: deck: java.lang.NoClassDefFoundError: net/customware/confluence/plugin/composition/CompositionUtil
Error formatting macro: card: java.lang.NoClassDefFoundError: net/customware/confluence/plugin/composition/CompositionUtil
<?xml version="1.0"?>
<!DOCTYPE cache PUBLIC
    "-//GemStone Systems, Inc.//GemFire Declarative Caching 6.5//EN"
    "http://www.gemstone.com/dtd/cache6_5.dtd">

<!--
  | DataOverflow.xml
  |
  | Configures a region to overflow to disk when the region reaches 
  | a certain capacity. The data files are written to the subdirectories 
  | specified in the disk-dirs elements.  
-->
<cache>
    <disk-store name="ds1" auto-compact="true" max-oplog-size="1024" queue-size="10000" time-interval="15">
	<disk-dirs>
		<disk-dir dir-size="4096">overflowData1</disk-dir>
	</disk-dirs>
    </disk-store>
	<region name="exampleRegion">
		<region-attributes disk-store-name="ds1" disk-synchronous="false">
			<!-- Write buffer data to disk every 10Kb or 15 milliseconds,
			     whichever comes first -->

			<eviction-attributes>
				<lru-memory-size maximum="1" action="overflow-to-disk" />
			</eviction-attributes>
		</region-attributes>
	</region>
</cache>
Error formatting macro: card: java.lang.NoClassDefFoundError: net/customware/confluence/plugin/composition/CompositionUtil
package quickstart;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;

import com.gemstone.gemfire.cache.Cache;
import com.gemstone.gemfire.cache.CacheFactory;
import com.gemstone.gemfire.cache.DiskStore;
import com.gemstone.gemfire.cache.Region;

/**
 * This example shows cached data overflow to disk. Overflow is used to keep a region size in check without completely
 * destroying the data. It is specified as the eviction action of a Least Recently Used (LRU) eviction controller
 * installed on the exampleRegion. Please refer to the quickstart guide for instructions on how to run this example.
 *
 * @author GemStone Systems, Inc.
 * @since 4.1.1
 */
public class DataOverflow {

	private final BufferedReader inputReader;

	private String overflowDirString;
	private Cache cache;
	private Region<String, byte[]> exampleRegion;

	public static void main(String[] args) throws Exception {

		DataOverflow dataOverflowExample = new DataOverflow();

		dataOverflowExample.createOverflowDirectory();
		dataOverflowExample.initialize();
		dataOverflowExample.causeOverflow();
		dataOverflowExample.destroyOverflowRegion();
		dataOverflowExample.cleanup();

		System.exit(0);
	}

	public DataOverflow() {
		this.inputReader = new BufferedReader(new InputStreamReader(System.in));
	}

	private void createOverflowDirectory() {
		System.out.println("This example uses disk to extend a region's capacity. The region is");
		System.out.println("configured with an eviction controller that overflows data to disk when");
		System.out.println("the region reaches a specified capacity.");

		File dir = new File("overflowData1");
		dir.mkdir();
	}

	private void initialize() throws IOException {
		System.out.println();
		System.out.println("Connecting to the distributed system and creating the cache.");

		// Create the cache. This causes the cache-xml-file to be parsed.
		this.cache = new CacheFactory()
                  .set("name", "DataOverflow")
                  .set("cache-xml-file", "xml/DataOverflow.xml")
                  .create();

		// Get the exampleRegion
		this.exampleRegion = cache.getRegion("exampleRegion");

		System.out.println();
		System.out.println("Example region \"" + exampleRegion.getFullPath() + "\" created in cache. ");

		// Get overflowDir from attributes of exampleRegion.
		String diskStoreName = exampleRegion.getAttributes().getDiskStoreName();
		DiskStore ds = this.cache.findDiskStore(diskStoreName);
		if (ds == null) {
		  return;
		}
		File[] overflowDirs = ds.getDiskDirs();
		overflowDirString = "";
		for (int i = 0; i < overflowDirs.length; i++) {
			if (i > 0) {
				overflowDirString += ", ";
			}
			overflowDirString += overflowDirs[i];
		}
	}

	private void causeOverflow() throws IOException {

		System.out.println();
		System.out.println("Putting 150 cache entries of 10 kilobytes each into the cache. When the");
		System.out.println("configured limit of 1 megabyte capacity is reached, the data will overflow");
		System.out.println("to files in " + overflowDirString + ".");

		for (long i = 0; i < 150; i++) {
			byte[] array = new byte[10 * 1024]; // size approximately 10 KB
			exampleRegion.put("key" + i, array);
		}

		System.out.println();
		System.out.println("Finished putting entries.");
		System.out.println();
		System.out.println("Use another shell to see the overflow files in " + overflowDirString + ".");
		System.out.println("The disk is used to extend available memory and these files are");
		System.out.println("treated as part of the local cache.");
		System.out.println();

		pressEnterToContinue();
	}

	private void destroyOverflowRegion() throws IOException {
		System.out.println();
		System.out.println("Destroying exampleRegion...");
		exampleRegion.destroyRegion();

		System.out.println();
		System.out.println("Please look again in " + overflowDirString + ". The overflow files are");
		System.out.println("deleted when the region is destroyed.");
		System.out.println();

		pressEnterToContinue();
	}

	private void cleanup() {
		System.out.println();
		System.out.println("Closing the cache and disconnecting.");

		cache.close();
	}

	private void pressEnterToContinue() throws IOException {
		System.out.println("Press Enter in this shell to continue.");
		inputReader.readLine();
	}
}

Related Javadocs

EvictionAttributes, DiskStoreFactory, DiskStore

Enter labels to add to this page:
Please wait 
Looking for a label? Just start typing.