/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.io.hfile;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.params.provider.Arguments;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;

/**
 * A kind of integration test at the intersection of {@link HFileBlock}, {@link CacheConfig}, and
 * {@link LruBlockCache}.
 */
@Tag(IOTests.TAG)
@Tag(SmallTests.TAG)
@HBaseParameterizedTestTemplate(name = "{index}: cacheOnWrite={0}")
public class TestLazyDataBlockDecompression {

  private static final Logger LOG = LoggerFactory.getLogger(TestLazyDataBlockDecompression.class);
  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  private static final Random RNG = new Random(9713312); // Just a fixed seed.

  private FileSystem fs;

  private final boolean cacheOnWrite;

  public static Stream<Arguments> parameters() {
    return Stream.of(Arguments.of(false), Arguments.of(true));
  }

  public TestLazyDataBlockDecompression(boolean cacheOnWrite) {
    this.cacheOnWrite = cacheOnWrite;
  }

  @BeforeEach
  public void setUp() throws IOException {
    fs = FileSystem.get(TEST_UTIL.getConfiguration());
  }

  @AfterEach
  public void tearDown() {
    fs = null;
  }

  /**
   * Write {@code entryCount} random keyvalues to a new HFile at {@code path}. Returns the row bytes
   * of the KeyValues written, in the order they were written.
   */
  private static void writeHFile(Configuration conf, CacheConfig cc, FileSystem fs, Path path,
    HFileContext cxt, int entryCount) throws IOException {
    HFile.Writer writer =
      new HFile.WriterFactory(conf, cc).withPath(fs, path).withFileContext(cxt).create();

    // write a bunch of random kvs
    final byte[] family = Bytes.toBytes("f");
    final byte[] qualifier = Bytes.toBytes("q");
    for (int i = 0; i < entryCount; i++) {
      byte[] keyBytes = RandomKeyValueUtil.randomOrderedKey(RNG, i);
      byte[] valueBytes = RandomKeyValueUtil.randomValue(RNG);
      // make a real keyvalue so that hfile tool can examine it
      writer.append(new KeyValue(keyBytes, family, qualifier, valueBytes));
    }
    writer.close();
  }

  /**
   * Read all blocks from {@code path} to populate {@code blockCache}.
   */
  private static void cacheBlocks(Configuration conf, CacheConfig cacheConfig, FileSystem fs,
    Path path, HFileContext cxt) throws IOException {
    FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path);
    long fileSize = fs.getFileStatus(path).getLen();
    FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize);
    ReaderContext context = new ReaderContextBuilder().withFilePath(path).withFileSize(fileSize)
      .withFileSystem(fsdis.getHfs()).withInputStreamWrapper(fsdis).build();
    HFileInfo fileInfo = new HFileInfo(context, conf);
    HFile.Reader reader = new HFilePreadReader(context, fileInfo, cacheConfig, conf);
    fileInfo.initMetaAndIndex(reader);
    long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset();
    List<HFileBlock> blocks = new ArrayList<>(4);
    HFileBlock block;
    while (offset <= max) {
      block = reader.readBlock(offset, -1, /* cacheBlock */ true, /* pread */ false,
        /* isCompaction */ false, /* updateCacheMetrics */ true, null, null);
      offset += block.getOnDiskSizeWithHeader();
      blocks.add(block);
    }
    LOG.info("read " + Iterables.toString(blocks));
    reader.close();
  }

  @TestTemplate
  public void testCompressionIncreasesEffectiveBlockCacheSize() throws Exception {
    // enough room for 2 uncompressed block
    int maxSize = (int) (HConstants.DEFAULT_BLOCKSIZE * 2.1);
    Path hfilePath =
      new Path(TEST_UTIL.getDataTestDir(), "testCompressionIncreasesEffectiveBlockcacheSize");
    HFileContext context =
      new HFileContextBuilder().withCompression(Compression.Algorithm.GZ).build();
    LOG.info("context=" + context);

    // setup cache with lazy-decompression disabled.
    Configuration lazyCompressDisabled = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
    lazyCompressDisabled.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, cacheOnWrite);
    lazyCompressDisabled.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, cacheOnWrite);
    lazyCompressDisabled.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, cacheOnWrite);
    lazyCompressDisabled.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, false);
    CacheConfig cc = new CacheConfig(lazyCompressDisabled,
      new LruBlockCache(maxSize, HConstants.DEFAULT_BLOCKSIZE, false, lazyCompressDisabled));
    assertFalse(cc.shouldCacheDataCompressed());
    assertFalse(cc.isCombinedBlockCache());
    LruBlockCache disabledBlockCache = (LruBlockCache) cc.getBlockCache().get();
    LOG.info("disabledBlockCache=" + disabledBlockCache);
    assertEquals(maxSize, disabledBlockCache.getMaxSize(), "test inconsistency detected.");
    assertTrue(disabledBlockCache.getEvictionThread() == null,
      "eviction thread spawned unintentionally.");
    assertEquals(0, disabledBlockCache.getBlockCount(),
      "freshly created blockcache contains blocks.");

    // 2000 kv's is ~3.6 full unencoded data blocks.
    // Requires a conf and CacheConfig but should not be specific to this instance's cache settings
    writeHFile(lazyCompressDisabled, cc, fs, hfilePath, context, 2000);

    // populate the cache
    cacheBlocks(lazyCompressDisabled, cc, fs, hfilePath, context);
    long disabledBlockCount = disabledBlockCache.getBlockCount();
    assertTrue(disabledBlockCount > 0,
      "blockcache should contain blocks. disabledBlockCount=" + disabledBlockCount);
    long disabledEvictedCount = disabledBlockCache.getStats().getEvictedCount();
    for (Map.Entry<BlockCacheKey, LruCachedBlock> e : disabledBlockCache.getMapForTests()
      .entrySet()) {
      HFileBlock block = (HFileBlock) e.getValue().getBuffer();
      assertTrue(block.isUnpacked(), "found a packed block, block=" + block);
    }

    // count blocks with lazy decompression
    Configuration lazyCompressEnabled = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
    lazyCompressEnabled.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, cacheOnWrite);
    lazyCompressEnabled.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, cacheOnWrite);
    lazyCompressEnabled.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, cacheOnWrite);
    lazyCompressEnabled.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, true);
    cc = new CacheConfig(lazyCompressEnabled,
      new LruBlockCache(maxSize, HConstants.DEFAULT_BLOCKSIZE, false, lazyCompressEnabled));
    assertTrue(cc.shouldCacheDataCompressed(), "test improperly configured.");
    assertTrue(cc.getBlockCache().get() instanceof LruBlockCache);
    LruBlockCache enabledBlockCache = (LruBlockCache) cc.getBlockCache().get();
    LOG.info("enabledBlockCache=" + enabledBlockCache);
    assertEquals(maxSize, enabledBlockCache.getMaxSize(), "test inconsistency detected");
    assertTrue(enabledBlockCache.getEvictionThread() == null,
      "eviction thread spawned unintentionally.");
    assertEquals(0, enabledBlockCache.getBlockCount(),
      "freshly created blockcache contains blocks.");

    cacheBlocks(lazyCompressEnabled, cc, fs, hfilePath, context);
    long enabledBlockCount = enabledBlockCache.getBlockCount();
    assertTrue(enabledBlockCount > 0,
      "blockcache should contain blocks. enabledBlockCount=" + enabledBlockCount);
    long enabledEvictedCount = enabledBlockCache.getStats().getEvictedCount();
    int candidatesFound = 0;
    for (Map.Entry<BlockCacheKey, LruCachedBlock> e : enabledBlockCache.getMapForTests()
      .entrySet()) {
      candidatesFound++;
      HFileBlock block = (HFileBlock) e.getValue().getBuffer();
      if (cc.shouldCacheCompressed(block.getBlockType().getCategory())) {
        assertFalse(block.isUnpacked(), "found an unpacked block, block=" + block
          + ", block buffer capacity=" + block.getBufferWithoutHeader().capacity());
      }
    }
    assertTrue(candidatesFound > 0,
      "did not find any candidates for compressed caching. Invalid test.");

    LOG.info(
      "disabledBlockCount=" + disabledBlockCount + ", enabledBlockCount=" + enabledBlockCount);
    assertTrue(disabledBlockCount < enabledBlockCount,
      "enabling compressed data blocks should increase the effective cache size. "
        + "disabledBlockCount=" + disabledBlockCount + ", enabledBlockCount=" + enabledBlockCount);

    LOG.info("disabledEvictedCount=" + disabledEvictedCount + ", enabledEvictedCount="
      + enabledEvictedCount);
    assertTrue(enabledEvictedCount < disabledEvictedCount,
      "enabling compressed data blocks should reduce the number of evictions. "
        + "disabledEvictedCount=" + disabledEvictedCount + ", enabledEvictedCount="
        + enabledEvictedCount);
  }
}
