/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.io;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionInputStream;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

public class TestMapFile {
  private static final Logger LOG = LoggerFactory.getLogger(TestMapFile.class);
  private static final Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
      TestMapFile.class.getSimpleName()));
  
  private static Configuration conf = new Configuration();

  @BeforeEach
  public void setup() throws Exception {
    LocalFileSystem fs = FileSystem.getLocal(conf);
    if (fs.exists(TEST_DIR) && !fs.delete(TEST_DIR, true)) {
      fail("Can't clean up test root dir");
    }
    fs.mkdirs(TEST_DIR);
  }
  
  private static final Progressable defaultProgressable = new Progressable() {
    @Override
    public void progress() {
    }
  };

  private static final CompressionCodec defaultCodec = new CompressionCodec() {
    @Override
    public CompressionOutputStream createOutputStream(OutputStream out)
        throws IOException {
      return mock(CompressionOutputStream.class);
    }

    @Override
    public CompressionOutputStream createOutputStream(OutputStream out,
        Compressor compressor) throws IOException {
      return mock(CompressionOutputStream.class);
    }

    @Override
    public Class<? extends Compressor> getCompressorType() {
      return null;
    }

    @Override
    public Compressor createCompressor() {
      return null;
    }

    @Override
    public CompressionInputStream createInputStream(InputStream in)
        throws IOException {
      return null;
    }

    @Override
    public CompressionInputStream createInputStream(InputStream in,
        Decompressor decompressor) throws IOException {
      return null;
    }

    @Override
    public Class<? extends Decompressor> getDecompressorType() {
      return null;
    }

    @Override
    public Decompressor createDecompressor() {
      return null;
    }

    @Override
    public String getDefaultExtension() {
      return null;
    }
  };

  private MapFile.Writer createWriter(String fileName,
      Class<? extends WritableComparable<?>> keyClass,
      Class<? extends Writable> valueClass) throws IOException {
    Path dirName = new Path(TEST_DIR, fileName);
    MapFile.Writer.setIndexInterval(conf, 4);
    return new MapFile.Writer(conf, dirName, MapFile.Writer.keyClass(keyClass),
        MapFile.Writer.valueClass(valueClass));
  }

  private MapFile.Reader createReader(String fileName,
      Class<? extends WritableComparable<?>> keyClass) throws IOException {
    Path dirName = new Path(TEST_DIR, fileName);
    return new MapFile.Reader(dirName, conf,
        MapFile.Reader.comparator(new WritableComparator(keyClass)));
  }
  
  /**
   * test {@code MapFile.Reader.getClosest()} method 
   *
   */
  @Test
  public void testGetClosestOnCurrentApi() throws Exception {
    final String TEST_PREFIX = "testGetClosestOnCurrentApi.mapfile";
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer = createWriter(TEST_PREFIX, Text.class, Text.class);
      int FIRST_KEY = 1;
      // Test keys: 11,21,31,...,91
      for (int i = FIRST_KEY; i < 100; i += 10) {      
        Text t = new Text(Integer.toString(i));
        writer.append(t, t);
      }
      writer.close();

      reader = createReader(TEST_PREFIX, Text.class);
      Text key = new Text("55");
      Text value = new Text();

      // Test get closest with step forward
      Text closest = (Text) reader.getClosest(key, value);
      assertEquals(new Text("61"), closest);

      // Test get closest with step back
      closest = (Text) reader.getClosest(key, value, true);
      assertEquals(new Text("51"), closest);

      // Test get closest when we pass explicit key
      final Text explicitKey = new Text("21");
      closest = (Text) reader.getClosest(explicitKey, value);
      assertEquals(new Text("21"), explicitKey);

      // Test what happens at boundaries. Assert if searching a key that is
      // less than first key in the mapfile, that the first key is returned.
      key = new Text("00");
      closest = (Text) reader.getClosest(key, value);
      assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));

      // Assert that null is returned if key is > last entry in mapfile.
      key = new Text("92");
      closest = (Text) reader.getClosest(key, value);
      assertNull(closest, "Not null key in testGetClosestWithNewCode");

      // If we were looking for the key before, we should get the last key
      closest = (Text) reader.getClosest(key, value, true);
      assertEquals(new Text("91"), closest);
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }
  
  /**
   * test {@code MapFile.Reader.midKey() } method 
   */
  @Test
  public void testMidKeyOnCurrentApi() throws Exception {
    // Write a mapfile of simple data: keys are
    final String TEST_PREFIX = "testMidKeyOnCurrentApi.mapfile";
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer = createWriter(TEST_PREFIX, IntWritable.class, IntWritable.class);
      // 0,1,....9
      int SIZE = 10;
      for (int i = 0; i < SIZE; i++)
        writer.append(new IntWritable(i), new IntWritable(i));
      writer.close();

      reader = createReader(TEST_PREFIX, IntWritable.class);
      assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey());
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }
  
  /**
   * test  {@code MapFile.Writer.rename()} method 
   */
  @Test
  public void testRename() {
    final String NEW_FILE_NAME = "test-new.mapfile";
    final String OLD_FILE_NAME = "test-old.mapfile";
    MapFile.Writer writer = null;
    try {
      FileSystem fs = FileSystem.getLocal(conf);
      writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
      writer.close();
      MapFile.rename(fs, new Path(TEST_DIR, OLD_FILE_NAME).toString(), 
          new Path(TEST_DIR, NEW_FILE_NAME).toString());
      MapFile.delete(fs, new Path(TEST_DIR, NEW_FILE_NAME).toString());
    } catch (IOException ex) {
      fail("testRename error " + ex);
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }
  
  /**
   * test {@code MapFile.rename()} 
   *  method with throwing {@code IOException}  
   */
  @Test
  public void testRenameWithException() {
    final String ERROR_MESSAGE = "Can't rename file";
    final String NEW_FILE_NAME = "test-new.mapfile";
    final String OLD_FILE_NAME = "test-old.mapfile";
    MapFile.Writer writer = null;
    try {
      FileSystem fs = FileSystem.getLocal(conf);
      FileSystem spyFs = spy(fs);

      writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
      writer.close();

      Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
      Path newDir = new Path(TEST_DIR, NEW_FILE_NAME);
      when(spyFs.rename(oldDir, newDir)).thenThrow(
          new IOException(ERROR_MESSAGE));

      MapFile.rename(spyFs, oldDir.toString(), newDir.toString());
      fail("testRenameWithException no exception error !!!");
    } catch (IOException ex) {
      assertEquals(ex.getMessage(), ERROR_MESSAGE,
          "testRenameWithException invalid IOExceptionMessage !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }

  @Test
  public void testRenameWithFalse() {
    final String ERROR_MESSAGE = "Could not rename";
    final String NEW_FILE_NAME = "test-new.mapfile";
    final String OLD_FILE_NAME = "test-old.mapfile";
    MapFile.Writer writer = null;
    try {
      FileSystem fs = FileSystem.getLocal(conf);
      FileSystem spyFs = spy(fs);

      writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
      writer.close();

      Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
      Path newDir = new Path(TEST_DIR, NEW_FILE_NAME);
      when(spyFs.rename(oldDir, newDir)).thenReturn(false);

      MapFile.rename(spyFs, oldDir.toString(), newDir.toString());
      fail("testRenameWithException no exception error !!!");
    } catch (IOException ex) {
      assertTrue(ex.getMessage().startsWith(ERROR_MESSAGE),
          "testRenameWithFalse invalid IOExceptionMessage error !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }
  
  /**
   * test throwing {@code IOException} in {@code MapFile.Writer} constructor    
   */
  @Test
  public void testWriteWithFailDirCreation() {
    String ERROR_MESSAGE = "Mkdirs failed to create directory";
    Path dirName = new Path(TEST_DIR, "fail.mapfile");
    MapFile.Writer writer = null;
    try {
      FileSystem fs = FileSystem.getLocal(conf);
      FileSystem spyFs = spy(fs);
      Path pathSpy = spy(dirName);
      when(pathSpy.getFileSystem(conf)).thenReturn(spyFs);
      when(spyFs.mkdirs(dirName)).thenReturn(false);

      writer = new MapFile.Writer(conf, pathSpy,
          MapFile.Writer.keyClass(IntWritable.class),
          MapFile.Writer.valueClass(Text.class));
      fail("testWriteWithFailDirCreation error !!!");
    } catch (IOException ex) {
      assertTrue(ex.getMessage().startsWith(ERROR_MESSAGE),
          "testWriteWithFailDirCreation ex error !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }

  /**
   * test {@code MapFile.Reader.finalKey()} method
   */
  @Test
  public void testOnFinalKey() {
    final String TEST_METHOD_KEY = "testOnFinalKey.mapfile";
    int SIZE = 10;
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
        IntWritable.class);
      for (int i = 0; i < SIZE; i++)
        writer.append(new IntWritable(i), new IntWritable(i));
      writer.close();

      reader = createReader(TEST_METHOD_KEY, IntWritable.class);
      IntWritable expectedKey = new IntWritable(0);
      reader.finalKey(expectedKey);
      assertEquals(expectedKey, new IntWritable(9),
          "testOnFinalKey not same !!!");
    } catch (IOException ex) {
      fail("testOnFinalKey error !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }
  
  /**
   * test {@code MapFile.Writer} constructor with key, value
   * and validate it with {@code keyClass(), valueClass()} methods 
   */
  @Test
  public void testKeyValueClasses() {
    Class<? extends WritableComparable<?>> keyClass = IntWritable.class;
    Class<?> valueClass = Text.class;
    try {
      createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class)
        .close();
      assertNotNull(MapFile.Writer.keyClass(keyClass),
          "writer key class null error !!!");
      assertNotNull(MapFile.Writer.valueClass(valueClass),
          "writer value class null error !!!");
    } catch (IOException ex) {
      fail(ex.getMessage());
    }
  }
  
  /**
   * test {@code MapFile.Reader.getClosest() } with wrong class key
   */
  @Test
  public void testReaderGetClosest() throws Exception {
    final String TEST_METHOD_KEY = "testReaderWithWrongKeyClass.mapfile";
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);

      for (int i = 0; i < 10; i++)
        writer.append(new IntWritable(i), new Text("value" + i));
      writer.close();

      reader = createReader(TEST_METHOD_KEY, Text.class);
      reader.getClosest(new Text("2"), new Text(""));
      fail("no excepted exception in testReaderWithWrongKeyClass !!!");
    } catch (IOException ex) {
      /* Should be thrown to pass the test */
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }
  
  /**
   * test {@code MapFile.Writer.append() } with wrong key class
   */
  @Test
  public void testReaderWithWrongValueClass() {
    final String TEST_METHOD_KEY = "testReaderWithWrongValueClass.mapfile";
    MapFile.Writer writer = null;
    try {
      writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
      writer.append(new IntWritable(0), new IntWritable(0));
      fail("no excepted exception in testReaderWithWrongKeyClass !!!");
    } catch (IOException ex) {
      /* Should be thrown to pass the test */
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }
  
  /**
   * test {@code MapFile.Reader.next(key, value)} for iteration.
   */
  @Test
  public void testReaderKeyIteration() {
    final String TEST_METHOD_KEY = "testReaderKeyIteration.mapfile";
    int SIZE = 10;
    int ITERATIONS = 5;
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
      int start = 0;
      for (int i = 0; i < SIZE; i++)
        writer.append(new IntWritable(i), new Text("Value:" + i));
      writer.close();

      reader = createReader(TEST_METHOD_KEY, IntWritable.class);
      // test iteration
      Writable startValue = new Text("Value:" + start);
      int i = 0;
      while (i++ < ITERATIONS) {
        IntWritable key = new IntWritable(start);
        Writable value = startValue;
        while (reader.next(key, value)) {
          assertNotNull(key);
          assertNotNull(value);
        }
        reader.reset();
      }
      assertTrue(reader.seek(new IntWritable(SIZE / 2)),
          "reader seek error !!!");
      assertFalse(reader.seek(new IntWritable(SIZE * 2)),
          "reader seek error !!!");
    } catch (IOException ex) {
      fail("reader seek error !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }

  /**
   * test {@code MapFile.Writer.testFix} method
   */
  @Test
  public void testFix() {
    final String INDEX_LESS_MAP_FILE = "testFix.mapfile";
    int PAIR_SIZE = 20;
    MapFile.Writer writer = null;
    try {
      FileSystem fs = FileSystem.getLocal(conf);
      Path dir = new Path(TEST_DIR, INDEX_LESS_MAP_FILE);
      writer = createWriter(INDEX_LESS_MAP_FILE, IntWritable.class, Text.class);
      for (int i = 0; i < PAIR_SIZE; i++)
        writer.append(new IntWritable(0), new Text("value"));
      writer.close();

      File indexFile = new File(".", "." + INDEX_LESS_MAP_FILE + "/index");
      boolean isDeleted = false;
      if (indexFile.exists())
        isDeleted = indexFile.delete();

      if (isDeleted)
        assertTrue(MapFile.fix(fs, dir, IntWritable.class, Text.class, true, conf) == PAIR_SIZE,
            "testFix error !!!");
    } catch (Exception ex) {
      fail("testFix error !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }

  /**
   * test {@link MapFile#fix(FileSystem, Path, Class<? extends Writable>,
   *                         Class<? extends Writable>, boolean, Configuration)}
   * method in case of BLOCK compression
   */
  @Test
  public void testFixBlockCompress() throws Exception {
    final String indexLessMapFile = "testFixBlockCompress.mapfile";
    final int compressBlocksize = 100;
    final int indexInterval = 4;
    final int noBlocks = 4;
    final String value = "value-";
    final int size = noBlocks * compressBlocksize / (4 + value.length());

    conf.setInt("io.seqfile.compress.blocksize", compressBlocksize);
    MapFile.Writer.setIndexInterval(conf, indexInterval);
    FileSystem fs = FileSystem.getLocal(conf);
    Path dir = new Path(TEST_DIR, indexLessMapFile);
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer =
          new MapFile.Writer(conf, dir,
          MapFile.Writer.keyClass(IntWritable.class),
          MapFile.Writer.valueClass(Text.class),
          MapFile.Writer.compression(CompressionType.BLOCK));
      for (int i = 0; i < size; i++) {
        writer.append(new IntWritable(i), new Text(value + i));
      }
      writer.close();
      Path index = new Path(dir, MapFile.INDEX_FILE_NAME);
      fs.rename(index, index.suffix(".orig"));

      assertEquals(size, MapFile.fix(fs, dir, IntWritable.class, Text.class, false, conf),
          "No of valid MapFile entries wrong");
      reader = new MapFile.Reader(dir, conf);
      IntWritable key;
      Text val = new Text();
      int notFound = 0;
      for (int i = 0; i < size; i++) {
        key = new IntWritable(i);
        if (null == reader.get(key, val)) {
          notFound++;
        }
      }
      assertEquals(0, notFound,
          "With MapFile.fix-ed index, could not get entries # ");
    } finally {
      IOUtils.cleanupWithLogger(null, writer, reader);
      if (fs.exists(dir)) {
        fs.delete(dir, true);
      }
    }
  }

  /**
   * test all available constructor for {@code MapFile.Writer}
   */
  @Test
  @SuppressWarnings("deprecation")
  public void testDeprecatedConstructors() {
    String path = new Path(TEST_DIR, "writes.mapfile").toString();
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      FileSystem fs = FileSystem.getLocal(conf);
      writer = new MapFile.Writer(conf, fs, path,
          IntWritable.class, Text.class, CompressionType.RECORD);
      assertNotNull(writer);
      writer.close();
      writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
          Text.class, CompressionType.RECORD, defaultProgressable);
      assertNotNull(writer);
      writer.close();
      writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
          Text.class, CompressionType.RECORD, defaultCodec, defaultProgressable);
      assertNotNull(writer);
      writer.close();
      writer = new MapFile.Writer(conf, fs, path,
          WritableComparator.get(Text.class), Text.class);
      assertNotNull(writer);
      writer.close();
      writer = new MapFile.Writer(conf, fs, path,
          WritableComparator.get(Text.class), Text.class,
          SequenceFile.CompressionType.RECORD);
      assertNotNull(writer);
      writer.close();
      writer = new MapFile.Writer(conf, fs, path,
          WritableComparator.get(Text.class), Text.class,
          CompressionType.RECORD, defaultProgressable);
      assertNotNull(writer);
      writer.close();

      reader = new MapFile.Reader(fs, path,
          WritableComparator.get(IntWritable.class), conf);
      assertNotNull(reader);
      assertNotNull(reader.getKeyClass(), "reader key is null !!!");
      assertNotNull(reader.getValueClass(), "reader value in null");
    } catch (IOException e) {
      fail(e.getMessage());
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }
  
  /**
   * test {@code MapFile.Writer} constructor 
   * with IllegalArgumentException  
   *  
   */
  @Test
  public void testKeyLessWriterCreation() {
    MapFile.Writer writer = null;
    try {
      writer = new MapFile.Writer(conf, TEST_DIR);
      fail("fail in testKeyLessWriterCreation !!!");
    } catch (IllegalArgumentException ex) {
    } catch (Exception e) {
      fail("fail in testKeyLessWriterCreation. Other ex !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }
  /**
   * test {@code MapFile.Writer} constructor with IOException
   */
  @Test
  public void testPathExplosionWriterCreation() {
    Path path = new Path(TEST_DIR, "testPathExplosionWriterCreation.mapfile");
    String TEST_ERROR_MESSAGE = "Mkdirs failed to create directory "
        + path.getName();
    MapFile.Writer writer = null;
    try {
      FileSystem fsSpy = spy(FileSystem.get(conf));
      Path pathSpy = spy(path);
      when(fsSpy.mkdirs(path)).thenThrow(new IOException(TEST_ERROR_MESSAGE));

      when(pathSpy.getFileSystem(conf)).thenReturn(fsSpy);

      writer = new MapFile.Writer(conf, pathSpy,
          MapFile.Writer.keyClass(IntWritable.class),
          MapFile.Writer.valueClass(IntWritable.class));
      fail("fail in testPathExplosionWriterCreation !!!");
    } catch (IOException ex) {
      assertEquals(ex.getMessage(), TEST_ERROR_MESSAGE,
          "testPathExplosionWriterCreation ex message error !!!");
    } catch (Exception e) {
      fail("fail in testPathExplosionWriterCreation. Other ex !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }

  /**
   * test {@code MapFile.Writer.append} method with desc order  
   */
  @Test
  public void testDescOrderWithThrowExceptionWriterAppend() {
    MapFile.Writer writer = null;
    try {
      writer = createWriter(".mapfile", IntWritable.class, Text.class);
      writer.append(new IntWritable(2), new Text("value: " + 1));
      writer.append(new IntWritable(2), new Text("value: " + 2));
      writer.append(new IntWritable(2), new Text("value: " + 4));
      writer.append(new IntWritable(1), new Text("value: " + 3));
      fail("testDescOrderWithThrowExceptionWriterAppend not expected exception error !!!");
    } catch (IOException ex) {
    } catch (Exception e) {
      fail("testDescOrderWithThrowExceptionWriterAppend other ex throw !!!");
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer);
    }
  }

  @Test
  public void testMainMethodMapFile() {
    String inFile = "mainMethodMapFile.mapfile";
    String path = new Path(TEST_DIR, inFile).toString();
    String[] args = { path, path };
    MapFile.Writer writer = null;
    try {
      writer = createWriter(inFile, IntWritable.class, Text.class);
      writer.append(new IntWritable(1), new Text("test_text1"));
      writer.append(new IntWritable(2), new Text("test_text2"));
      writer.close();
      MapFile.main(args);
    } catch (Exception ex) {
      fail("testMainMethodMapFile error !!!");
    } finally {
      IOUtils.cleanupWithLogger(null, writer);
    }
  }

  /**
   * Test getClosest feature.
   * 
   * @throws Exception
   */
  @Test
  @SuppressWarnings("deprecation")
  public void testGetClosest() throws Exception {
    // Write a mapfile of simple data: keys are
    Path dirName = new Path(TEST_DIR, "testGetClosest.mapfile");
    FileSystem fs = FileSystem.getLocal(conf);
    Path qualifiedDirName = fs.makeQualified(dirName);
    // Make an index entry for every third insertion.
    MapFile.Writer.setIndexInterval(conf, 3);
    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
        Text.class, Text.class);
      // Assert that the index interval is 1
      assertEquals(3, writer.getIndexInterval());
      // Add entries up to 100 in intervals of ten.
      final int FIRST_KEY = 10;
      for (int i = FIRST_KEY; i < 100; i += 10) {
        String iStr = Integer.toString(i);
        Text t = new Text("00".substring(iStr.length()) + iStr);
        writer.append(t, t);
      }
      writer.close();
      // Now do getClosest on created mapfile.
      reader = new MapFile.Reader(qualifiedDirName, conf);
      Text key = new Text("55");
      Text value = new Text();
      Text closest = (Text) reader.getClosest(key, value);
      // Assert that closest after 55 is 60
      assertEquals(new Text("60"), closest);
      // Get closest that falls before the passed key: 50
      closest = (Text) reader.getClosest(key, value, true);
      assertEquals(new Text("50"), closest);
      // Test get closest when we pass explicit key
      final Text TWENTY = new Text("20");
      closest = (Text) reader.getClosest(TWENTY, value);
      assertEquals(TWENTY, closest);
      closest = (Text) reader.getClosest(TWENTY, value, true);
      assertEquals(TWENTY, closest);
      // Test what happens at boundaries. Assert if searching a key that is
      // less than first key in the mapfile, that the first key is returned.
      key = new Text("00");
      closest = (Text) reader.getClosest(key, value);
      assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));

      // If we're looking for the first key before, and we pass in a key before
      // the first key in the file, we should get null
      closest = (Text) reader.getClosest(key, value, true);
      assertNull(closest);

      // Assert that null is returned if key is > last entry in mapfile.
      key = new Text("99");
      closest = (Text) reader.getClosest(key, value);
      assertNull(closest);

      // If we were looking for the key before, we should get the last key
      closest = (Text) reader.getClosest(key, value, true);
      assertEquals(new Text("90"), closest);
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }

  @Test
  @SuppressWarnings("deprecation")
  public void testMidKey() throws Exception {
    // Write a mapfile of simple data: keys are
    Path dirName = new Path(TEST_DIR, "testMidKey.mapfile");
    FileSystem fs = FileSystem.getLocal(conf);
    Path qualifiedDirName = fs.makeQualified(dirName);

    MapFile.Writer writer = null;
    MapFile.Reader reader = null;
    try {
      writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
        IntWritable.class, IntWritable.class);
      writer.append(new IntWritable(1), new IntWritable(1));
      writer.close();
      // Now do getClosest on created mapfile.
      reader = new MapFile.Reader(qualifiedDirName, conf);
      assertEquals(new IntWritable(1), reader.midKey());
    } finally {
      IOUtils.cleanupWithLogger(LOG, writer, reader);
    }
  }

  @Test
  @SuppressWarnings("deprecation")
  public void testMidKeyEmpty() throws Exception {
    // Write a mapfile of simple data: keys are
    Path dirName = new Path(TEST_DIR, "testMidKeyEmpty.mapfile");
    FileSystem fs = FileSystem.getLocal(conf);
    Path qualifiedDirName = fs.makeQualified(dirName);

    MapFile.Writer writer = new MapFile.Writer(conf, fs,
        qualifiedDirName.toString(), IntWritable.class, IntWritable.class);
    writer.close();
    // Now do getClosest on created mapfile.
    MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
    try {
      assertEquals(null, reader.midKey()); 
    } finally {
      reader.close();
    }
  }

  @Test
  public void testMerge() throws Exception {
    final String TEST_METHOD_KEY = "testMerge.mapfile";
    int SIZE = 10;
    int ITERATIONS = 5;
    Path[] in = new Path[5];
    List<Integer> expected = new ArrayList<Integer>();
    for (int j = 0; j < 5; j++) {
      try (MapFile.Writer writer = createWriter(TEST_METHOD_KEY + "." + j,
          IntWritable.class, Text.class)) {
        in[j] = new Path(TEST_DIR, TEST_METHOD_KEY + "." + j);
        for (int i = 0; i < SIZE; i++) {
          expected.add(i + j);
          writer.append(new IntWritable(i + j), new Text("Value:" + (i + j)));
        }
      }
    }
    // Sort expected values
    Collections.sort(expected);
    // Merge all 5 files
    MapFile.Merger merger = new MapFile.Merger(conf);
    merger.merge(in, true, new Path(TEST_DIR, TEST_METHOD_KEY));

    try (MapFile.Reader reader = createReader(TEST_METHOD_KEY,
        IntWritable.class)) {
      int start = 0;
      // test iteration
      Text startValue = new Text("Value:" + start);
      int i = 0;
      while (i++ < ITERATIONS) {
        Iterator<Integer> expectedIterator = expected.iterator();
        IntWritable key = new IntWritable(start);
        Text value = startValue;
        IntWritable prev = new IntWritable(start);
        while (reader.next(key, value)) {
          assertTrue(prev.get() <= key.get(),
              "Next key should be always equal or more");
          assertEquals(expectedIterator.next().intValue(), key.get());
          prev.set(key.get());
        }
        reader.reset();
      }
    }

    // inputs should be deleted
    for (int j = 0; j < in.length; j++) {
      Path path = in[j];
      assertFalse(path.getFileSystem(conf).exists(path),
          "inputs should be deleted");
    }
  }
}
