/*
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */
package org.apache.bookkeeper.bookie;

import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_ENTRY_LOG_COUNT;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ACTIVE_ENTRY_LOG_SPACE_BYTES;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MAJOR_COMPACTION_COUNT;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MINOR_COMPACTION_COUNT;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_COMPACTION_SPACE_BYTES;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_DELETION_SPACE_BYTES;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.THREAD_RUNTIME;
import static org.apache.bookkeeper.bookie.TransactionalEntryLogCompactor.COMPACTED_SUFFIX;
import static org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import com.google.common.util.concurrent.UncheckedExecutionException;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.buffer.UnpooledByteBufAllocator;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import org.apache.bookkeeper.bookie.BookieException.EntryLogMetadataMapException;
import org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException;
import org.apache.bookkeeper.bookie.storage.CompactionEntryLog;
import org.apache.bookkeeper.bookie.storage.ldb.PersistentEntryLogMetadataMap;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
import org.apache.bookkeeper.meta.LedgerManager;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.LedgerMetadataListener;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.Processor;
import org.apache.bookkeeper.proto.checksum.DigestManager;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.apache.bookkeeper.test.TestStatsProvider;
import org.apache.bookkeeper.util.DiskChecker;
import org.apache.bookkeeper.util.TestUtils;
import org.apache.bookkeeper.versioning.Version;
import org.apache.bookkeeper.versioning.Versioned;
import org.apache.zookeeper.AsyncCallback;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This class tests the entry log compaction functionality.
 */
public abstract class CompactionTest extends BookKeeperClusterTestCase {

    private static final Logger LOG = LoggerFactory.getLogger(CompactionTest.class);

    private static final int ENTRY_SIZE = 1024;
    private static final int NUM_BOOKIES = 1;

    private final boolean isThrottleByBytes;
    private final DigestType digestType;
    private final byte[] passwdBytes;
    private final int numEntries;
    private final int gcWaitTime;
    private final double minorCompactionThreshold;
    private final double majorCompactionThreshold;
    private final long minorCompactionInterval;
    private final long majorCompactionInterval;
    private final String msg;
    private final boolean useMetadataCache;

    public CompactionTest(boolean isByBytes, boolean useMetadataCache) {
        super(NUM_BOOKIES);

        this.isThrottleByBytes = isByBytes;
        this.useMetadataCache = useMetadataCache;
        this.digestType = DigestType.CRC32;
        this.passwdBytes = "".getBytes();
        numEntries = 100;
        gcWaitTime = 1000;
        minorCompactionThreshold = 0.1f;
        majorCompactionThreshold = 0.5f;
        minorCompactionInterval = 2 * gcWaitTime / 1000;
        majorCompactionInterval = 4 * gcWaitTime / 1000;

        // a dummy message
        StringBuilder msgSB = new StringBuilder();
        for (int i = 0; i < ENTRY_SIZE; i++) {
            msgSB.append("a");
        }
        msg = msgSB.toString();
    }

    @Before
    @Override
    public void setUp() throws Exception {
        // Set up the configuration properties needed.
        baseConf.setEntryLogSizeLimit(numEntries * ENTRY_SIZE);
        // Disable skip list for compaction
        baseConf.setGcWaitTime(gcWaitTime);
        baseConf.setFlushInterval(100);
        baseConf.setMinorCompactionThreshold(minorCompactionThreshold);
        baseConf.setMajorCompactionThreshold(majorCompactionThreshold);
        baseConf.setMinorCompactionInterval(minorCompactionInterval);
        baseConf.setMajorCompactionInterval(majorCompactionInterval);
        baseConf.setEntryLogFilePreAllocationEnabled(false);
        baseConf.setLedgerStorageClass(InterleavedLedgerStorage.class.getName());
        baseConf.setIsThrottleByBytes(this.isThrottleByBytes);
        baseConf.setIsForceGCAllowWhenNoSpace(false);
        baseConf.setGcEntryLogMetadataCacheEnabled(useMetadataCache);
        super.setUp();
    }

    private GarbageCollectorThread getGCThread() throws Exception {
        assertEquals(1, bookieCount());
        BookieServer server = serverByIndex(0);
        return ((InterleavedLedgerStorage) server.getBookie().getLedgerStorage()).gcThread;
    }

    LedgerHandle[] prepareData(int numEntryLogs, boolean changeNum)
        throws Exception {
        // since an entry log file can hold at most 100 entries
        // first ledger write 2 entries, which is less than low water mark
        int num1 = 2;
        // third ledger write more than high water mark entries
        int num3 = (int) (numEntries * 0.7f);
        // second ledger write remaining entries, which is higher than low water mark
        // and less than high water mark
        int num2 = numEntries - num3 - num1;

        LedgerHandle[] lhs = new LedgerHandle[3];
        for (int i = 0; i < 3; ++i) {
            lhs[i] = bkc.createLedger(NUM_BOOKIES, NUM_BOOKIES, digestType, passwdBytes);
        }

        for (int n = 0; n < numEntryLogs; n++) {
            for (int k = 0; k < num1; k++) {
                lhs[0].addEntry(msg.getBytes());
            }
            for (int k = 0; k < num2; k++) {
                lhs[1].addEntry(msg.getBytes());
            }
            for (int k = 0; k < num3; k++) {
                lhs[2].addEntry(msg.getBytes());
            }
            if (changeNum) {
                --num2;
                ++num3;
            }
        }

        return lhs;
    }

    private void verifyLedger(long lid, long startEntryId, long endEntryId) throws Exception {
        LedgerHandle lh = bkc.openLedger(lid, digestType, passwdBytes);
        Enumeration<LedgerEntry> entries = lh.readEntries(startEntryId, endEntryId);
        while (entries.hasMoreElements()) {
            LedgerEntry entry = entries.nextElement();
            assertEquals(msg, new String(entry.getEntry()));
        }
    }

    @Test
    public void testDisableCompaction() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        // disable compaction
        // restart bookies
        restartBookies(c -> {
                c.setMinorCompactionThreshold(0.0f);
                c.setMajorCompactionThreshold(0.0f);
                return c;
            });

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;

        // remove ledger2 and ledger3
        // so entry log 1 and 2 would have ledger1 entries left
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());
        LOG.info("Finished deleting the ledgers contains most entries.");

        assertFalse(getGCThread().enableMajorCompaction);
        assertFalse(getGCThread().enableMinorCompaction);
        getGCThread().triggerGC().get();
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }

        // after garbage collection, compaction should not be executed
        assertEquals(lastMinorCompactionTime, getGCThread().lastMinorCompactionTime);
        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);

        // entry logs ([0,1].log) should not be compacted.
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertTrue("Not Found entry log file ([0,1].log that should have been compacted in ledgerDirectory: "
                            + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, false, 0, 1));
        }
    }

    @Test
    public void testForceGarbageCollectionWhenDisableCompactionConfigurationSettings() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        restartBookies(c -> {
            c.setForceAllowCompaction(true);
            c.setMajorCompactionThreshold(0.5f);
            c.setMinorCompactionThreshold(0.2f);
            c.setMajorCompactionInterval(0);
            c.setMinorCompactionInterval(0);
            return c;
        });

        assertFalse(getGCThread().enableMajorCompaction);
        assertFalse(getGCThread().enableMinorCompaction);
        assertTrue(getGCThread().isForceMajorCompactionAllow);
        assertTrue(getGCThread().isForceMinorCompactionAllow);

        assertEquals(0.5f, getGCThread().majorCompactionThreshold, 0f);
        assertEquals(0.2f, getGCThread().minorCompactionThreshold, 0f);
    }

    @Test
    public void testForceGarbageCollection() throws Exception {
        testForceGarbageCollection(true);
        testForceGarbageCollection(false);
    }

    public void testForceGarbageCollection(boolean isForceCompactionAllowWhenDisableCompaction) throws Exception {
        ServerConfiguration conf = newServerConfiguration();
        conf.setGcWaitTime(60000);
        if (isForceCompactionAllowWhenDisableCompaction) {
            conf.setMinorCompactionInterval(0);
            conf.setMajorCompactionInterval(0);
            conf.setForceAllowCompaction(true);
            conf.setMajorCompactionThreshold(0.5f);
            conf.setMinorCompactionThreshold(0.2f);
        } else {
            conf.setMinorCompactionInterval(120000);
            conf.setMajorCompactionInterval(240000);
        }
        LedgerDirsManager dirManager = new LedgerDirsManager(conf, conf.getLedgerDirs(),
                new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
        CheckpointSource cp = new CheckpointSource() {
            @Override
            public Checkpoint newCheckpoint() {
                // Do nothing.
                return null;
            }

            @Override
            public void checkpointComplete(Checkpoint checkPoint, boolean compact)
                throws IOException {
                // Do nothing.
            }
        };
        for (File journalDir : conf.getJournalDirs()) {
            BookieImpl.checkDirectoryStructure(journalDir);
        }
        for (File dir : dirManager.getAllLedgerDirs()) {
            BookieImpl.checkDirectoryStructure(dir);
        }
        runFunctionWithLedgerManagerFactory(conf, lmf -> {
            try (LedgerManager lm = lmf.newLedgerManager()) {
                InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
                storage.initialize(
                    conf,
                    lm,
                    dirManager,
                    dirManager,
                    NullStatsLogger.INSTANCE,
                    UnpooledByteBufAllocator.DEFAULT);
                storage.setCheckpointSource(cp);
                storage.setCheckpointer(Checkpointer.NULL);

                storage.start();
                long startTime = System.currentTimeMillis();
                storage.gcThread.enableForceGC();
                storage.gcThread.triggerGC().get(); //major
                storage.gcThread.triggerGC().get(); //minor
                // Minor and Major compaction times should be larger than when we started
                // this test.
                assertTrue("Minor or major compaction did not trigger even on forcing.",
                    storage.gcThread.lastMajorCompactionTime > startTime
                        && storage.gcThread.lastMinorCompactionTime > startTime);
                storage.shutdown();
            } catch (Exception e) {
                throw new UncheckedExecutionException(e.getMessage(), e);
            }
            return null;
        });
    }

    @Test
    public void testForceGarbageCollectionWhenDiskIsFull() throws Exception {
        testForceGarbageCollectionWhenDiskIsFull(true);
        testForceGarbageCollectionWhenDiskIsFull(false);
    }

    public void testForceGarbageCollectionWhenDiskIsFull(boolean isForceCompactionAllowWhenDisableCompaction)
        throws Exception {

        restartBookies(conf -> {
            if (isForceCompactionAllowWhenDisableCompaction) {
                conf.setMinorCompactionInterval(0);
                conf.setMajorCompactionInterval(0);
                conf.setForceAllowCompaction(true);
                conf.setMajorCompactionThreshold(0.5f);
                conf.setMinorCompactionThreshold(0.2f);
            } else {
                conf.setMinorCompactionInterval(120000);
                conf.setMajorCompactionInterval(240000);
            }
            return conf;
        });

        getGCThread().suspendMajorGC();
        getGCThread().suspendMinorGC();
        long majorCompactionCntBeforeGC = 0;
        long minorCompactionCntBeforeGC = 0;
        long majorCompactionCntAfterGC = 0;
        long minorCompactionCntAfterGC = 0;

        // disable forceMajor and forceMinor
        majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        getGCThread().triggerGC(true, true, true).get();
        majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        assertEquals(majorCompactionCntBeforeGC, majorCompactionCntAfterGC);
        assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);

        // enable forceMajor and forceMinor
        majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        getGCThread().triggerGC(true, false, false).get();
        majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        assertEquals(majorCompactionCntBeforeGC + 1, majorCompactionCntAfterGC);
        assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);

        // enable forceMajor and disable forceMinor
        majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        getGCThread().triggerGC(true, false, true).get();
        majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        assertEquals(majorCompactionCntBeforeGC + 1, majorCompactionCntAfterGC);
        assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);

        // disable forceMajor and enable forceMinor
        majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        getGCThread().triggerGC(true, true, false).get();
        majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
        minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
        assertEquals(majorCompactionCntBeforeGC, majorCompactionCntAfterGC);
        assertEquals(minorCompactionCntBeforeGC + 1, minorCompactionCntAfterGC);

    }

    @Test
    public void testMinorCompaction() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c -> {
                // disable major compaction
                c.setMajorCompactionThreshold(0.0f);
                c.setGcWaitTime(60000);
                c.setMinorCompactionInterval(120000);
                c.setMajorCompactionInterval(240000);
                return c;
            });


        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }
        assertTrue(
                "ACTIVE_ENTRY_LOG_COUNT should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_COUNT)
                        .getSample().intValue() > 0);
        assertTrue(
                "ACTIVE_ENTRY_LOG_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_SPACE_BYTES)
                        .getSample().intValue() > 0);

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertTrue(getGCThread().enableMinorCompaction);

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();

        // after garbage collection, major compaction should not be executed
        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);

        // entry logs ([0,1,2].log) should be compacted.
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                    + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even though entry log files are removed, we still can access entries for ledger1
        // since those entries have been compacted to a new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());

        assertTrue(
                "RECLAIMED_COMPACTION_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getCounter("bookie.gc." + RECLAIMED_COMPACTION_SPACE_BYTES)
                        .get().intValue() > 0);
        assertTrue(
                "RECLAIMED_DELETION_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getCounter("bookie.gc." + RECLAIMED_DELETION_SPACE_BYTES)
                        .get().intValue() > 0);
    }

    @Test
    public void testMinorCompactionWithMaxTimeMillisOk() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(6, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // disable major compaction
        // restart bookies
        restartBookies(c-> {
            c.setMajorCompactionThreshold(0.0f);
            c.setGcWaitTime(60000);
            c.setMinorCompactionInterval(120000);
            c.setMajorCompactionInterval(240000);

            // Setup limit on compaction duration.
            // The limit is enough to compact.
            c.setMinorCompactionMaxTimeMillis(5000);
            c.setMajorCompactionMaxTimeMillis(5000);

            return c;
        });

        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        assertTrue(
                "ACTIVE_ENTRY_LOG_COUNT should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_COUNT)
                        .getSample().intValue() > 0);
        assertTrue(
                "ACTIVE_ENTRY_LOG_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_SPACE_BYTES)
                        .getSample().intValue() > 0);

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertTrue(getGCThread().enableMinorCompaction);

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();

        // after garbage collection, major compaction should not be executed
        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);

        // entry logs ([0,1,2].log) should be compacted.
        for (File ledgerDirectory : tmpDirs.getDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                            + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even though entry log files are removed, we still can access entries for ledger1
        // since those entries have been compacted to a new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());

        assertTrue(
                "RECLAIMED_COMPACTION_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getCounter("bookie.gc." + RECLAIMED_COMPACTION_SPACE_BYTES)
                        .get().intValue() > 0);
        assertTrue(
                "RECLAIMED_DELETION_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getCounter("bookie.gc." + RECLAIMED_DELETION_SPACE_BYTES)
                        .get().intValue() > 0);
    }


    @Test
    public void testMinorCompactionWithMaxTimeMillisTooShort() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(6, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // disable major compaction
        // restart bookies
        restartBookies(c-> {
            c.setMajorCompactionThreshold(0.0f);
            c.setGcWaitTime(60000);
            c.setMinorCompactionInterval(120000);
            c.setMajorCompactionInterval(240000);

            // Setup limit on compaction duration.
            // The limit is not enough to finish the compaction
            c.setMinorCompactionMaxTimeMillis(1);
            c.setMajorCompactionMaxTimeMillis(1);

            return c;
        });

        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        assertTrue(
                "ACTIVE_ENTRY_LOG_COUNT should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_COUNT)
                        .getSample().intValue() > 0);
        assertTrue(
                "ACTIVE_ENTRY_LOG_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_SPACE_BYTES)
                        .getSample().intValue() > 0);

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertTrue(getGCThread().enableMinorCompaction);

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();

        // after garbage collection, major compaction should not be executed
        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);

        // entry logs ([0,1,2].log) should be compacted.
        for (File ledgerDirectory : tmpDirs.getDirs()) {
            // Compaction of at least one of the files should not finish up
            assertTrue("Not found entry log file ([0,1,2].log that should not have been compacted in ledgerDirectory: "
                    + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());
    }

    @Test
    public void testForceMinorCompaction() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c-> {
            c.setMajorCompactionThreshold(0.0f);
            c.setGcWaitTime(60000);
            c.setMinorCompactionInterval(-1);
            c.setMajorCompactionInterval(-1);
            c.setForceAllowCompaction(true);
            return c;
        });

        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        assertTrue(
                "ACTIVE_ENTRY_LOG_COUNT should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_COUNT)
                        .getSample().intValue() > 0);
        assertTrue(
                "ACTIVE_ENTRY_LOG_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_SPACE_BYTES)
                        .getSample().intValue() > 0);

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertFalse(getGCThread().enableMinorCompaction);

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();

        // after garbage collection, major compaction should not be executed
        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);

        // entry logs ([0,1,2].log) should be compacted.
        for (File ledgerDirectory : tmpDirs.getDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                    + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even though entry log files are removed, we still can access entries for ledger1
        // since those entries have been compacted to a new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());

        assertTrue(
                "RECLAIMED_COMPACTION_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getCounter("bookie.gc." + RECLAIMED_COMPACTION_SPACE_BYTES)
                        .get().intValue() > 0);
        assertTrue(
                "RECLAIMED_DELETION_SPACE_BYTES should have been updated",
                getStatsProvider(0)
                        .getCounter("bookie.gc." + RECLAIMED_DELETION_SPACE_BYTES)
                        .get().intValue() > 0);
    }

    @Test
    public void testMinorCompactionWithEntryLogPerLedgerEnabled() throws Exception {
        // restart bookies
        restartBookies(c-> {
            c.setMajorCompactionThreshold(0.0f);
            c.setGcWaitTime(60000);
            c.setMinorCompactionInterval(120000);
            c.setMajorCompactionInterval(240000);
            c.setForceAllowCompaction(true);
            c.setEntryLogPerLedgerEnabled(true);
            return c;
        });

        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertTrue(getGCThread().enableMinorCompaction);

        // remove ledgers 1 and 2
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        // Need to wait until entry log 3 gets flushed before initiating GC to satisfy assertions.
        while (!getGCThread().entryLogger.getFlushedLogIds().contains(3L)) {
            TimeUnit.MILLISECONDS.sleep(100);
        }

        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().triggerGC(true, false, false).get();

        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);

        // At this point, we have the following state of ledgers end entry logs:
        // L0 (not deleted) -> E0 (un-flushed): Entry log should exist.
        // L1 (deleted) -> E1 (un-flushed): Entry log should exist as un-flushed entry logs are not considered for GC.
        // L2 (deleted) -> E2 (flushed): Entry log should have been garbage collected.
        //                 E3 (flushed): Entry log should have been garbage collected.
        //                 E4 (un-flushed): Entry log should exist as un-flushed entry logs are not considered for GC.
        assertTrue("Not found entry log files [0, 1, 4].log that should not have been compacted in: "
                + tmpDirs.getDirs().get(0), TestUtils.hasAllLogFiles(tmpDirs.getDirs().get(0), 0, 1, 4));
        assertTrue("Found entry log files [2, 3].log that should have been compacted in ledgerDirectory: "
                + tmpDirs.getDirs().get(0), TestUtils.hasNoneLogFiles(tmpDirs.getDirs().get(0), 2, 3));

        // Now, let's mark E1 as flushed, as its ledger L1 has been deleted already. In this case, the GC algorithm
        // should consider it for deletion.
        ((DefaultEntryLogger) getGCThread().entryLogger).recentlyCreatedEntryLogsStatus.flushRotatedEntryLog(1L);
        getGCThread().triggerGC(true, false, false).get();
        assertTrue("Found entry log file 1.log that should have been compacted in ledgerDirectory: "
                + tmpDirs.getDirs().get(0), TestUtils.hasNoneLogFiles(tmpDirs.getDirs().get(0), 1));

        // Once removed the ledger L0, then deleting E0 is fine (only if it has been flushed).
        bkc.deleteLedger(lhs[0].getId());
        getGCThread().triggerGC(true, false, false).get();
        assertTrue("Found entry log file 0.log that should not have been compacted in ledgerDirectory: "
                + tmpDirs.getDirs().get(0), TestUtils.hasAllLogFiles(tmpDirs.getDirs().get(0), 0));
        ((DefaultEntryLogger) getGCThread().entryLogger).recentlyCreatedEntryLogsStatus.flushRotatedEntryLog(0L);
        getGCThread().triggerGC(true, false, false).get();
        assertTrue("Found entry log file 0.log that should have been compacted in ledgerDirectory: "
                + tmpDirs.getDirs().get(0), TestUtils.hasNoneLogFiles(tmpDirs.getDirs().get(0), 0));
    }

    @Test
    public void testMinorCompactionWithNoWritableLedgerDirs() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c -> {
                // disable major compaction
                c.setMajorCompactionThreshold(0.0f);
                c.setGcWaitTime(60000);
                c.setMinorCompactionInterval(120000);
                c.setMajorCompactionInterval(240000);
                return c;
            });

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertTrue(getGCThread().enableMinorCompaction);

        for (int i = 0; i < bookieCount(); i++) {
            BookieImpl bookie = ((BookieImpl) serverByIndex(i).getBookie());
            LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
            List<File> ledgerDirs = ledgerDirsManager.getAllLedgerDirs();
            // if all the discs are full then Major and Minor compaction would be disabled since
            // 'isForceGCAllowWhenNoSpace' is not enabled. Check LedgerDirsListener of interleavedLedgerStorage.
            for (File ledgerDir : ledgerDirs) {
                ledgerDirsManager.addToFilledDirs(ledgerDir);
            }
        }

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().triggerGC().get();
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }
        // after garbage collection, major compaction should not be executed
        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
        assertEquals(lastMinorCompactionTime, getGCThread().lastMinorCompactionTime);

        // entry logs ([0,1,2].log) should still remain, because both major and Minor compaction are disabled.
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertTrue(
                    "All the entry log files ([0,1,2].log are not available, which is not expected" + ledgerDirectory,
                    TestUtils.hasLogFiles(ledgerDirectory, false, 0, 1, 2));
        }
    }

    @Test
    public void testMinorCompactionWithNoWritableLedgerDirsButIsForceGCAllowWhenNoSpaceIsSet() throws Exception {
        stopAllBookies();
        ServerConfiguration conf = newServerConfiguration();
        // disable major compaction
        conf.setMajorCompactionThreshold(0.0f);
        // here we are setting isForceGCAllowWhenNoSpace to true, so Major and Minor compaction wont be disabled in case
        // when discs are full
        conf.setIsForceGCAllowWhenNoSpace(true);
        conf.setGcWaitTime(600000);
        conf.setMinorCompactionInterval(120000);
        conf.setMajorCompactionInterval(240000);
        // We need at least 2 ledger dirs because compaction will flush ledger cache, and will
        // trigger relocateIndexFileAndFlushHeader. If we only have one ledger dir, compaction will always fail
        // when there's no writeable ledger dir.
        File ledgerDir1 = tmpDirs.createNew("ledger", "test1");
        File ledgerDir2 = tmpDirs.createNew("ledger", "test2");
        File journalDir = tmpDirs.createNew("journal", "test");
        String[] ledgerDirNames = new String[]{
            ledgerDir1.getPath(),
            ledgerDir2.getPath()
        };
        conf.setLedgerDirNames(ledgerDirNames);
        conf.setJournalDirName(journalDir.getPath());
        BookieServer server = startAndAddBookie(conf).getServer();
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertTrue(getGCThread().enableMinorCompaction);
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }
        for (int i = 0; i < bookieCount(); i++) {
            BookieImpl bookie = ((BookieImpl) serverByIndex(i).getBookie());
            bookie.getLedgerStorage().flush();
            bookie.dirsMonitor.shutdown();
            LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
            List<File> ledgerDirs = ledgerDirsManager.getAllLedgerDirs();
            // Major and Minor compaction are not disabled even though discs are full. Check LedgerDirsListener of
            // interleavedLedgerStorage.
            for (File ledgerDir : ledgerDirs) {
                ledgerDirsManager.addToFilledDirs(ledgerDir);
            }
        }

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().triggerGC(true, false, false).get();

        // after garbage collection, major compaction should not be executed
        assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);

        // though all discs are added to filled dirs list, compaction would succeed, because in EntryLogger for
        // allocating newlog
        // we get getWritableLedgerDirsForNewLog() of ledgerDirsManager instead of getWritableLedgerDirs()
        // entry logs ([0,1,2].log) should be compacted.
        for (File ledgerDirectory : ((BookieImpl) server.getBookie()).getLedgerDirsManager().getAllLedgerDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                    + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory.getParentFile(), true, 0, 1, 2));
        }

        // even entry log files are removed, we still can access entries for ledger1
        // since those entries has been compacted to new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());

        // for the sake of validity of test lets make sure that there is no writableLedgerDir in the bookies
        for (int i = 0; i < bookieCount(); i++) {
            BookieImpl bookie = (BookieImpl) serverByIndex(i).getBookie();
            LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
            try {
                List<File> ledgerDirs = ledgerDirsManager.getWritableLedgerDirs();
                // it is expected not to have any writableLedgerDirs since we added all of them to FilledDirs
                fail("It is expected not to have any writableLedgerDirs");
            } catch (NoWritableLedgerDirException nwe) {

            }
        }
    }

    @Test
    public void testMajorCompaction() throws Exception {

        // prepare data
        LedgerHandle[] lhs = prepareData(3, true);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c -> {
                // disable minor compaction
                c.setMinorCompactionThreshold(0.0f);
                c.setGcWaitTime(60000);
                c.setMinorCompactionInterval(120000);
                c.setMajorCompactionInterval(240000);
                return c;
            });

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertTrue(getGCThread().enableMajorCompaction);
        assertFalse(getGCThread().enableMinorCompaction);

        // remove ledger1 and ledger3
        bkc.deleteLedger(lhs[0].getId());
        bkc.deleteLedger(lhs[2].getId());
        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }
        // after garbage collection, minor compaction should not be executed
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);
        assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime);

        // entry logs ([0,1,2].log) should be compacted
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                      + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even entry log files are removed, we still can access entries for ledger2
        // since those entries has been compacted to new entry log
        verifyLedger(lhs[1].getId(), 0, lhs[1].getLastAddConfirmed());
    }

    @Test
    public void testForceMajorCompaction() throws Exception {

        // prepare data
        LedgerHandle[] lhs = prepareData(3, true);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c-> {
            // disable minor compaction
            c.setMinorCompactionThreshold(0.0f);
            c.setGcWaitTime(60000);
            c.setMinorCompactionInterval(-1);
            c.setMajorCompactionInterval(-1);
            c.setForceAllowCompaction(true);
            return c;
        });

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertFalse(getGCThread().enableMajorCompaction);
        assertFalse(getGCThread().enableMinorCompaction);
        assertTrue(getGCThread().isForceMajorCompactionAllow);
        assertFalse(getGCThread().isForceMinorCompactionAllow);

        // remove ledger1 and ledger3
        bkc.deleteLedger(lhs[0].getId());
        bkc.deleteLedger(lhs[2].getId());
        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }
        // after garbage collection, minor compaction should not be executed
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);
        assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime);

        // entry logs ([0,1,2].log) should be compacted
        for (File ledgerDirectory : tmpDirs.getDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                    + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even entry log files are removed, we still can access entries for ledger2
        // since those entries has been compacted to new entry log
        verifyLedger(lhs[1].getId(), 0, lhs[1].getLastAddConfirmed());
    }

    @Test
    public void testCompactionPersistence() throws Exception {
        /*
         * for this test scenario we are assuming that there will be only one
         * bookie in the cluster
         */
        assertEquals("Numbers of Bookies in this cluster", 1, numBookies);
        /*
         * this test is for validating EntryLogCompactor, so make sure
         * TransactionalCompaction is not enabled.
         */
        assertFalse("Bookies must be using EntryLogCompactor", baseConf.getUseTransactionalCompaction());
        // prepare data
        LedgerHandle[] lhs = prepareData(3, true);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c -> {
                // disable minor compaction
                c.setMinorCompactionThreshold(0.0f);
                c.setGcWaitTime(60000);
                c.setMinorCompactionInterval(120000);
                c.setMajorCompactionInterval(240000);
                return c;
            });

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertTrue(getGCThread().enableMajorCompaction);
        assertFalse(getGCThread().enableMinorCompaction);

        // remove ledger1 and ledger3
        bkc.deleteLedger(lhs[0].getId());
        bkc.deleteLedger(lhs[2].getId());
        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();

        // after garbage collection, minor compaction should not be executed
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);
        assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime);

        // entry logs ([0,1,2].log) should be compacted
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                    + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even entry log files are removed, we still can access entries for
        // ledger2
        // since those entries has been compacted to new entry log
        long ledgerId = lhs[1].getId();
        long lastAddConfirmed = lhs[1].getLastAddConfirmed();
        verifyLedger(ledgerId, 0, lastAddConfirmed);

        /*
         * there is only one bookie in the cluster so we should be able to read
         * entries from this bookie.
         */
        ServerConfiguration bookieServerConfig = ((BookieImpl) serverByIndex(0).getBookie()).conf;
        ServerConfiguration newBookieConf = new ServerConfiguration(bookieServerConfig);
        /*
         * by reusing bookieServerConfig and setting metadataServiceUri to null
         * we can create/start new Bookie instance using the same data
         * (journal/ledger/index) of the existing BookeieServer for our testing
         * purpose.
         */
        newBookieConf.setMetadataServiceUri(null);
        String entryLogCachePath = tmpDirs.createNew("entry", "bk2").getAbsolutePath();
        newBookieConf.setGcEntryLogMetadataCachePath(entryLogCachePath);
        Bookie newbookie = new TestBookieImpl(newBookieConf);

        DigestManager digestManager = DigestManager.instantiate(ledgerId, passwdBytes,
                BookKeeper.DigestType.toProtoDigestType(digestType), UnpooledByteBufAllocator.DEFAULT,
                baseClientConf.getUseV2WireProtocol());

        for (long entryId = 0; entryId <= lastAddConfirmed; entryId++) {
            ByteBuf readEntryBufWithChecksum = newbookie.readEntry(ledgerId, entryId);
            ByteBuf readEntryBuf = digestManager.verifyDigestAndReturnData(entryId, readEntryBufWithChecksum);
            byte[] readEntryBytes = new byte[readEntryBuf.readableBytes()];
            readEntryBuf.readBytes(readEntryBytes);
            assertEquals(msg, new String(readEntryBytes));
        }
    }

    @Test
    public void testCompactionWhenLedgerDirsAreFull() throws Exception {
        /*
         * for this test scenario we are assuming that there will be only one
         * bookie in the cluster
         */
        assertEquals("Numbers of Bookies in this cluster", 1, bookieCount());
        ServerConfiguration serverConfig = confByIndex(0);
        File ledgerDir = serverConfig.getLedgerDirs()[0];
        assertEquals("Number of Ledgerdirs for this bookie", 1, serverConfig.getLedgerDirs().length);
        assertTrue("indexdirs should be configured to null", null == serverConfig.getIndexDirs());
        /*
         * this test is for validating EntryLogCompactor, so make sure
         * TransactionalCompaction is not enabled.
         */
        assertFalse("Bookies must be using EntryLogCompactor", baseConf.getUseTransactionalCompaction());
        // prepare data
        LedgerHandle[] lhs = prepareData(3, true);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        serverByIndex(0).getBookie().getLedgerStorage().flush();
        assertTrue(
                "entry log file ([0,1,2].log should be available in ledgerDirectory: "
                        + serverConfig.getLedgerDirs()[0],
                TestUtils.hasLogFiles(serverConfig.getLedgerDirs()[0], false, 0, 1, 2));

        long usableSpace = ledgerDir.getUsableSpace();
        long totalSpace = ledgerDir.getTotalSpace();

        /*
         * because of the value set for diskUsageThreshold, when bookie is
         * restarted it wouldn't find any writableledgerdir. But we have set
         * very low values for minUsableSizeForEntryLogCreation and
         * minUsableSizeForIndexFileCreation, so it should be able to create
         * EntryLog file and Index file for doing compaction.
         */

        // restart bookies
        restartBookies(c -> {
                c.setForceReadOnlyBookie(true);
                c.setIsForceGCAllowWhenNoSpace(true);
                // disable minor compaction
                c.setMinorCompactionThreshold(0.0f);
                c.setGcWaitTime(60000);
                c.setMinorCompactionInterval(120000);
                c.setMajorCompactionInterval(240000);
                c.setMinUsableSizeForEntryLogCreation(1);
                c.setMinUsableSizeForIndexFileCreation(1);
                c.setDiskUsageThreshold((1.0f - ((float) usableSpace / (float) totalSpace)) * 0.9f);
                c.setDiskUsageWarnThreshold(0.0f);
                return c;
            });

        assertFalse("There shouldn't be any writable ledgerDir",
                    ((BookieImpl) serverByIndex(0).getBookie()).getLedgerDirsManager().hasWritableLedgerDirs());

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertTrue(getGCThread().enableMajorCompaction);
        assertFalse(getGCThread().enableMinorCompaction);

        // remove ledger1 and ledger3
        bkc.deleteLedger(lhs[0].getId());
        bkc.deleteLedger(lhs[2].getId());
        LOG.info("Finished deleting the ledgers contains most entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }
        // after garbage collection, minor compaction should not be executed
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);
        assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime);

        /*
         * GarbageCollection should have succeeded, so no previous entrylog
         * should be available.
         */

        // entry logs ([0,1,2].log) should be compacted
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
                    + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even entry log files are removed, we still can access entries for
        // ledger2
        // since those entries has been compacted to new entry log
        long ledgerId = lhs[1].getId();
        long lastAddConfirmed = lhs[1].getLastAddConfirmed();
        verifyLedger(ledgerId, 0, lastAddConfirmed);
    }

    @Test
    public void testMajorCompactionAboveThreshold() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
        long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
        assertTrue(getGCThread().enableMajorCompaction);
        assertTrue(getGCThread().enableMinorCompaction);

        // remove ledger1 and ledger2
        bkc.deleteLedger(lhs[0].getId());
        bkc.deleteLedger(lhs[1].getId());
        LOG.info("Finished deleting the ledgers contains less entries.");
        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();

        // after garbage collection, minor compaction should not be executed
        assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);
        assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime);

        // entry logs ([0,1,2].log) should not be compacted
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertTrue("Not Found entry log file ([1,2].log that should have been compacted in ledgerDirectory: "
                     + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, false, 0, 1, 2));
        }
    }

    @Test
    public void testCompactionSmallEntryLogs() throws Exception {

        // create a ledger to write a few entries
        LedgerHandle alh = bkc.createLedger(NUM_BOOKIES, NUM_BOOKIES, digestType, "".getBytes());
        for (int i = 0; i < 3; i++) {
           alh.addEntry(msg.getBytes());
        }
        alh.close();

        // restart bookie to roll entry log files
        restartBookies();

        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());
        LOG.info("Finished deleting the ledgers contains most entries.");
        // restart bookies again to roll entry log files.
        restartBookies();

        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();
        if (useMetadataCache) {
            assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap);
        }
        // entry logs (0.log) should not be compacted
        // entry logs ([1,2,3].log) should be compacted.
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertTrue("Not Found entry log file ([0].log that should have been compacted in ledgerDirectory: "
                     + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0));
            assertFalse("Found entry log file ([1,2,3].log that should have not been compacted in ledgerDirectory: "
                      + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 1, 2, 3));
        }

        // even entry log files are removed, we still can access entries for ledger1
        // since those entries has been compacted to new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());
    }

    /**
     * Test that compaction doesnt add to index without having persisted
     * entrylog first. This is needed because compaction doesn't go through the journal.
     * {@see https://issues.apache.org/jira/browse/BOOKKEEPER-530}
     * {@see https://issues.apache.org/jira/browse/BOOKKEEPER-664}
     */
    @Test
    public void testCompactionSafety() throws Exception {
        tearDown(); // I dont want the test infrastructure
        ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
        final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
        LedgerManager manager = getLedgerManager(ledgers);

        File tmpDir = tmpDirs.createNew("bkTest", ".dir");
        File curDir = BookieImpl.getCurrentDirectory(tmpDir);
        BookieImpl.checkDirectoryStructure(curDir);
        conf.setLedgerDirNames(new String[] {tmpDir.toString()});

        conf.setEntryLogSizeLimit(DefaultEntryLogger.LOGFILE_HEADER_SIZE + 3 * (4 + ENTRY_SIZE));
        conf.setGcWaitTime(100);
        conf.setMinorCompactionThreshold(0.7f);
        conf.setMajorCompactionThreshold(0.0f);
        conf.setMinorCompactionInterval(1);
        conf.setMajorCompactionInterval(10);
        conf.setPageLimit(1);

        CheckpointSource checkpointSource = new CheckpointSource() {
                AtomicInteger idGen = new AtomicInteger(0);
                class MyCheckpoint implements CheckpointSource.Checkpoint {
                    int id = idGen.incrementAndGet();
                    @Override
                    public int compareTo(CheckpointSource.Checkpoint o) {
                        if (o == CheckpointSource.Checkpoint.MAX) {
                            return -1;
                        } else if (o == CheckpointSource.Checkpoint.MIN) {
                            return 1;
                        }
                        return id - ((MyCheckpoint) o).id;
                    }
                }

                @Override
                public CheckpointSource.Checkpoint newCheckpoint() {
                    return new MyCheckpoint();
                }

                public void checkpointComplete(CheckpointSource.Checkpoint checkpoint, boolean compact)
                        throws IOException {
                }
            };
        final byte[] key = "foobar".getBytes();
        File log0 = new File(curDir, "0.log");
        LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs(),
                new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
        assertFalse("Log shouldnt exist", log0.exists());
        InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
        storage.initialize(
            conf,
            manager,
            dirs,
            dirs,
            NullStatsLogger.INSTANCE,
            UnpooledByteBufAllocator.DEFAULT);
        storage.setCheckpointSource(checkpointSource);
        storage.setCheckpointer(Checkpointer.NULL);

        ledgers.add(1L);
        ledgers.add(2L);
        ledgers.add(3L);
        storage.setMasterKey(1, key);
        storage.setMasterKey(2, key);
        storage.setMasterKey(3, key);
        storage.addEntry(genEntry(1, 1, ENTRY_SIZE));
        storage.addEntry(genEntry(2, 1, ENTRY_SIZE));
        storage.addEntry(genEntry(2, 2, ENTRY_SIZE));
        storage.addEntry(genEntry(3, 2, ENTRY_SIZE));
        storage.flush();
        storage.shutdown();

        assertTrue("Log should exist", log0.exists());
        ledgers.remove(2L);
        ledgers.remove(3L);

        storage = new InterleavedLedgerStorage();
        storage.initialize(
            conf,
            manager,
            dirs, dirs,
            NullStatsLogger.INSTANCE,
            UnpooledByteBufAllocator.DEFAULT);
        storage.setCheckpointSource(checkpointSource);
        storage.setCheckpointer(Checkpointer.NULL);

        storage.start();
        for (int i = 0; i < 10; i++) {
            if (!log0.exists()) {
                break;
            }
            Thread.sleep(1000);
            storage.entryLogger.flush(); // simulate sync thread
        }
        assertFalse("Log shouldnt exist", log0.exists());

        ledgers.add(4L);
        storage.setMasterKey(4, key);
        storage.addEntry(genEntry(4, 1, ENTRY_SIZE)); // force ledger 1 page to flush
        storage.shutdown();

        storage = new InterleavedLedgerStorage();
        storage.initialize(
            conf,
            manager,
            dirs,
            dirs,
            NullStatsLogger.INSTANCE,
            UnpooledByteBufAllocator.DEFAULT);
        storage.setCheckpointSource(checkpointSource);
        storage.setCheckpointer(Checkpointer.NULL);

        storage.getEntry(1, 1); // entry should exist
    }

    @Test
    public void testCancelledCompactionWhenShuttingDown() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        // change compaction in low throughput
        // restart bookies
        restartBookies(c -> {
            c.setIsThrottleByBytes(true);
            c.setCompactionRateByBytes(ENTRY_SIZE / 1000);
            c.setMinorCompactionThreshold(0.2f);
            c.setMajorCompactionThreshold(0.5f);
            return c;
        });

        // remove ledger2 and ledger3
        // so entry log 1 and 2 would have ledger1 entries left
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());
        LOG.info("Finished deleting the ledgers contains most entries.");

        getGCThread().triggerGC(true, false, false);
        getGCThread().throttler.cancelledAcquire();
        waitUntilTrue(() -> {
            try {
                return getGCThread().compacting.get();
            } catch (Exception e) {
                fail("Get GC thread failed");
            }
            return null;
        }, () -> "Not attempting to complete", 10000, 200);

        getGCThread().shutdown();
        // after garbage collection shutdown, compaction should be cancelled when acquire permits
        // and GC running flag should be false.
        assertFalse(getGCThread().running);

    }

    private void waitUntilTrue(Supplier<Boolean> condition,
                               Supplier<String> msg,
                               long waitTime,
                               long pause) throws InterruptedException {
        long startTime = System.currentTimeMillis();
        while (true) {
            if (condition.get()) {
                return;
            }
            if (System.currentTimeMillis() > startTime + waitTime) {
                fail(msg.get());
            }
            Thread.sleep(Math.min(waitTime, pause));
        }
    }

    private LedgerManager getLedgerManager(final Set<Long> ledgers) {
        LedgerManager manager = new LedgerManager() {
                @Override
                public CompletableFuture<Versioned<LedgerMetadata>> createLedgerMetadata(long lid,
                                                                                         LedgerMetadata metadata) {
                    unsupported();
                    return null;
                }
                @Override
                public CompletableFuture<Void> removeLedgerMetadata(long ledgerId, Version version) {
                    unsupported();
                    return null;
                }
                @Override
                public CompletableFuture<Versioned<LedgerMetadata>> readLedgerMetadata(long ledgerId) {
                    unsupported();
                    return null;
                }
                @Override
                public CompletableFuture<Versioned<LedgerMetadata>> writeLedgerMetadata(long ledgerId,
                                                                                        LedgerMetadata metadata,
                                                                                        Version currentVersion) {
                    unsupported();
                    return null;
                }
                @Override
                public void asyncProcessLedgers(Processor<Long> processor,
                                                AsyncCallback.VoidCallback finalCb,
                        Object context, int successRc, int failureRc) {
                    unsupported();
                }
                @Override
                public void registerLedgerMetadataListener(long ledgerId,
                        LedgerMetadataListener listener) {
                    unsupported();
                }
                @Override
                public void unregisterLedgerMetadataListener(long ledgerId,
                        LedgerMetadataListener listener) {
                    unsupported();
                }
                @Override
                public void close() throws IOException {}

                void unsupported() {
                    LOG.error("Unsupported operation called", new Exception());
                    throw new RuntimeException("Unsupported op");
                }

                @Override
                public LedgerRangeIterator getLedgerRanges(long zkOpTimeoutMs) {
                    final AtomicBoolean hasnext = new AtomicBoolean(true);
                    return new LedgerManager.LedgerRangeIterator() {
                        @Override
                        public boolean hasNext() throws IOException {
                            return hasnext.get();
                        }
                        @Override
                        public LedgerManager.LedgerRange next() throws IOException {
                            hasnext.set(false);
                            return new LedgerManager.LedgerRange(ledgers);
                        }
                    };
                 }
            };
        return manager;
    }

    /**
     * Test that compaction should execute silently when there is no entry logs
     * to compact. {@see https://issues.apache.org/jira/browse/BOOKKEEPER-700}
     */
    @Test
    public void testWhenNoLogsToCompact() throws Exception {
        tearDown(); // I dont want the test infrastructure
        ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
        File tmpDir = tmpDirs.createNew("bkTest", ".dir");
        File curDir = BookieImpl.getCurrentDirectory(tmpDir);
        BookieImpl.checkDirectoryStructure(curDir);
        conf.setLedgerDirNames(new String[] { tmpDir.toString() });

        LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs(),
                new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
        final Set<Long> ledgers = Collections
                .newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
        LedgerManager manager = getLedgerManager(ledgers);
        CheckpointSource checkpointSource = new CheckpointSource() {

            @Override
            public Checkpoint newCheckpoint() {
                return null;
            }

            @Override
            public void checkpointComplete(Checkpoint checkpoint,
                    boolean compact) throws IOException {
            }
        };
        InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
        storage.initialize(
            conf,
            manager,
            dirs,
            dirs,
            NullStatsLogger.INSTANCE,
            UnpooledByteBufAllocator.DEFAULT);
        storage.setCheckpointSource(checkpointSource);
        storage.setCheckpointer(Checkpointer.NULL);

        double threshold = 0.1;
        long limit = 0;

        // shouldn't throw exception
        storage.gcThread.doCompactEntryLogs(threshold, limit);
    }

    private ByteBuf genEntry(long ledger, long entry, int size) {
        ByteBuf bb = Unpooled.buffer(size);
        bb.writeLong(ledger);
        bb.writeLong(entry);
        while (bb.isWritable()) {
            bb.writeByte((byte) 0xFF);
        }
        return bb;
    }

    /**
     * Suspend garbage collection when suspendMajor/suspendMinor is set.
     */
    @Test
    public void testSuspendGarbageCollection() throws Exception {
        ServerConfiguration conf = newServerConfiguration();
        conf.setGcWaitTime(500);
        conf.setMinorCompactionInterval(1);
        conf.setMajorCompactionInterval(2);
        conf.setMajorCompactionMaxTimeMillis(5000);
        runFunctionWithLedgerManagerFactory(conf, lmf -> {
            try (LedgerManager lm = lmf.newLedgerManager()) {
                testSuspendGarbageCollection(conf, lm);
            } catch (Exception e) {
                throw new UncheckedExecutionException(e.getMessage(), e);
            }
            return null;
        });
    }

    private void testSuspendGarbageCollection(ServerConfiguration conf,
                                              LedgerManager lm) throws Exception {
        LedgerDirsManager dirManager = new LedgerDirsManager(conf, conf.getLedgerDirs(),
                new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
        CheckpointSource cp = new CheckpointSource() {
            @Override
            public Checkpoint newCheckpoint() {
                // Do nothing.
                return null;
            }

            @Override
            public void checkpointComplete(Checkpoint checkPoint, boolean compact)
                throws IOException {
                // Do nothing.
            }
        };
        for (File journalDir : conf.getJournalDirs()) {
            BookieImpl.checkDirectoryStructure(journalDir);
        }
        for (File dir : dirManager.getAllLedgerDirs()) {
            BookieImpl.checkDirectoryStructure(dir);
        }
        InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
        TestStatsProvider stats = new TestStatsProvider();
        storage.initialize(
            conf,
            lm,
            dirManager,
            dirManager,
            stats.getStatsLogger("storage"),
            UnpooledByteBufAllocator.DEFAULT);
        storage.setCheckpointSource(cp);
        storage.setCheckpointer(Checkpointer.NULL);

        storage.start();

        int majorCompactions = stats.getCounter("storage.gc." + MAJOR_COMPACTION_COUNT).get().intValue();
        int minorCompactions = stats.getCounter("storage.gc." + MINOR_COMPACTION_COUNT).get().intValue();
        Thread.sleep(3 * (conf.getMajorCompactionInterval() * 1000
                + conf.getGcWaitTime()
                + conf.getMajorCompactionMaxTimeMillis()));
        assertTrue(
                "Major compaction should have happened",
                stats.getCounter("storage.gc." + MAJOR_COMPACTION_COUNT).get() > majorCompactions);

        // test suspend Major GC.
        storage.gcThread.suspendMajorGC();

        Thread.sleep(1000);
        long startTime = System.currentTimeMillis();
        majorCompactions = stats.getCounter("storage.gc." + MAJOR_COMPACTION_COUNT).get().intValue();
        Thread.sleep(conf.getMajorCompactionInterval() * 1000
                   + conf.getGcWaitTime());
        assertTrue("major compaction triggered while suspended",
                storage.gcThread.lastMajorCompactionTime < startTime);
        assertTrue("major compaction triggered while suspended",
                stats.getCounter("storage.gc." + MAJOR_COMPACTION_COUNT).get() == majorCompactions);

        // test suspend Major GC.
        Thread.sleep(conf.getMinorCompactionInterval() * 1000
                + conf.getGcWaitTime());
        assertTrue(
                "Minor compaction should have happened",
                stats.getCounter("storage.gc." + MINOR_COMPACTION_COUNT).get() > minorCompactions);

        // test suspend Minor GC.
        storage.gcThread.suspendMinorGC();

        Thread.sleep(1000);
        startTime = System.currentTimeMillis();
        minorCompactions = stats.getCounter("storage.gc." + MINOR_COMPACTION_COUNT).get().intValue();
        Thread.sleep(conf.getMajorCompactionInterval() * 1000
                   + conf.getGcWaitTime());
        assertTrue("minor compaction triggered while suspended",
                storage.gcThread.lastMinorCompactionTime < startTime);
        assertTrue("minor compaction triggered while suspended",
                stats.getCounter("storage.gc." + MINOR_COMPACTION_COUNT).get() == minorCompactions);

        // test resume
        storage.gcThread.resumeMinorGC();
        storage.gcThread.resumeMajorGC();

        Thread.sleep((conf.getMajorCompactionInterval() + conf.getMinorCompactionInterval()) * 1000
                + (conf.getGcWaitTime() * 2));
        assertTrue(
                "Major compaction should have happened",
                stats.getCounter("storage.gc." + MAJOR_COMPACTION_COUNT).get() > majorCompactions);
        assertTrue(
                "Minor compaction should have happened",
                stats.getCounter("storage.gc." + MINOR_COMPACTION_COUNT).get() > minorCompactions);
        assertTrue(
                "gcThreadRunttime should be non-zero",
                stats.getOpStatsLogger("storage.gc." + THREAD_RUNTIME).getSuccessCount() > 0);

    }

    @Test
    public void testRecoverIndexWhenIndexIsPartiallyFlush() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(3, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c -> {
                // disable compaction
                c.setMinorCompactionThreshold(0.0f);
                c.setMajorCompactionThreshold(0.0f);
                c.setGcWaitTime(600000);
                return c;
            });



        BookieImpl bookie = ((BookieImpl) serverByIndex(0).getBookie());
        InterleavedLedgerStorage storage = (InterleavedLedgerStorage) bookie.ledgerStorage;

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");

        MockTransactionalEntryLogCompactor partialCompactionWorker = new MockTransactionalEntryLogCompactor(
                ((InterleavedLedgerStorage) bookie.getLedgerStorage()).gcThread);

        for (long logId = 0; logId < 3; logId++) {
            EntryLogMetadata meta = storage.entryLogger.getEntryLogMetadata(logId);
            partialCompactionWorker.compactWithIndexFlushFailure(meta);
        }

        // entry logs ([0,1,2].log) should not be compacted because of partial flush throw IOException
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertTrue("Entry log file ([0,1,2].log should not be compacted in ledgerDirectory: "
                + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // entries should be available
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());

        // But we should see .compacted file with index flush failure
        assertEquals(findCompactedEntryLogFiles().size(), 3);

        // Now try to recover those flush failed index files
        partialCompactionWorker.cleanUpAndRecover();

        // There should be no .compacted files after recovery
        assertEquals(findCompactedEntryLogFiles().size(), 0);

        // compaction worker should recover partial flushed index and delete [0,1,2].log
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertFalse("Entry log file ([0,1,2].log should have been compacted in ledgerDirectory: "
                + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
        }

        // even entry log files are removed, we still can access entries for ledger1
        // since those entries has been compacted to new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());
    }

    @Test
    public void testCompactionFailureShouldNotResultInDuplicatedData() throws Exception {
        // prepare data
        LedgerHandle[] lhs = prepareData(5, false);

        for (LedgerHandle lh : lhs) {
            lh.close();
        }

        // restart bookies
        restartBookies(c -> {
                // disable compaction
                c.setMinorCompactionThreshold(0.0f);
                c.setMajorCompactionThreshold(0.0f);
                c.setUseTransactionalCompaction(true);
                return c;
            });

        // remove ledger2 and ledger3
        bkc.deleteLedger(lhs[1].getId());
        bkc.deleteLedger(lhs[2].getId());

        LOG.info("Finished deleting the ledgers contains most entries.");
        Thread.sleep(baseConf.getMajorCompactionInterval() * 1000
            + baseConf.getGcWaitTime());
        BookieImpl bookie = (BookieImpl) serverByIndex(0).getBookie();
        InterleavedLedgerStorage storage = (InterleavedLedgerStorage) bookie.ledgerStorage;

        List<File> ledgerDirs = bookie.getLedgerDirsManager().getAllLedgerDirs();
        List<Long> usageBeforeCompaction = new ArrayList<>();
        ledgerDirs.forEach(file -> usageBeforeCompaction.add(getDirectorySpaceUsage(file)));

        MockTransactionalEntryLogCompactor partialCompactionWorker = new MockTransactionalEntryLogCompactor(
            ((InterleavedLedgerStorage) bookie.ledgerStorage).gcThread);

        for (long logId = 0; logId < 5; logId++) {
            EntryLogMetadata meta = storage.entryLogger.getEntryLogMetadata(logId);
            partialCompactionWorker.compactWithLogFlushFailure(meta);
        }

        // entry logs ([0-4].log) should not be compacted because of failure in flush compaction log
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertTrue("Entry log file ([0,1,2].log should not be compacted in ledgerDirectory: "
                + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2, 3, 4));
        }
        // even entry log files are removed, we still can access entries for ledger1
        // since those entries has been compacted to new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());

        List<Long> freeSpaceAfterCompactionFailed = new ArrayList<>();
        ledgerDirs.forEach(file -> freeSpaceAfterCompactionFailed.add(getDirectorySpaceUsage(file)));

        // No extra data is generated after compaction fail
        for (int i = 0; i < usageBeforeCompaction.size(); i++) {
            assertEquals(usageBeforeCompaction.get(i), freeSpaceAfterCompactionFailed.get(i));
        }


        // restart bookies
        restartBookies(c -> {
                // now enable normal compaction
                c.setMajorCompactionThreshold(0.5f);
                c.setMajorCompactionMaxTimeMillis(5000);
                return c;
            });

        getGCThread().enableForceGC();
        getGCThread().triggerGC().get();

        Thread.sleep(confByIndex(0).getMajorCompactionInterval() * 1000
                + confByIndex(0).getGcWaitTime()
                + confByIndex(0).getMajorCompactionMaxTimeMillis());
        // compaction worker should compact [0-4].log
        for (File ledgerDirectory : bookieLedgerDirs()) {
            assertFalse("Entry log file ([0,1,2].log should have been compacted in ledgerDirectory: "
                + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2, 3, 4));
        }

        // even entry log files are removed, we still can access entries for ledger1
        // since those entries has been compacted to new entry log
        verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());
    }

    private long getDirectorySpaceUsage(File dir) {
        long size = 0;
        for (File file : dir.listFiles()) {
            size += file.length();
        }
        return size;
    }

    private Set<File> findCompactedEntryLogFiles() throws Exception {
        Set<File> compactedLogFiles = new HashSet<>();
        for (File ledgerDirectory : bookieLedgerDirs()) {
            File[] files = BookieImpl.getCurrentDirectory(ledgerDirectory).listFiles(
                file -> file.getName().endsWith(COMPACTED_SUFFIX));
            if (files != null) {
                Collections.addAll(compactedLogFiles, files);
            }
        }
        return compactedLogFiles;
    }

    private static class MockTransactionalEntryLogCompactor extends TransactionalEntryLogCompactor {

        public MockTransactionalEntryLogCompactor(GarbageCollectorThread gcThread) {
            super(gcThread.conf,
                  gcThread.entryLogger,
                  gcThread.ledgerStorage,
                  (long entry) -> {
                      try {
                          gcThread.removeEntryLog(entry);
                      } catch (EntryLogMetadataMapException e) {
                          LOG.warn("Failed to remove entry-log metadata {}", entry, e);
                      }
                  });
        }

        synchronized void compactWithIndexFlushFailure(EntryLogMetadata metadata) throws IOException {
            LOG.info("Compacting entry log {}.", metadata.getEntryLogId());
            CompactionEntryLog compactionLog = entryLogger.newCompactionLog(metadata.getEntryLogId());

            CompactionPhase scanEntryLog = new ScanEntryLogPhase(metadata, compactionLog);
            if (!scanEntryLog.run()) {
                LOG.info("Compaction for {} end in ScanEntryLogPhase.", metadata.getEntryLogId());
                return;
            }
            CompactionPhase flushCompactionLog = new FlushCompactionLogPhase(compactionLog);
            if (!flushCompactionLog.run()) {
                LOG.info("Compaction for {} end in FlushCompactionLogPhase.", metadata.getEntryLogId());
                return;
            }
            CompactionPhase partialFlushIndexPhase = new PartialFlushIndexPhase(compactionLog);
            if (!partialFlushIndexPhase.run()) {
                LOG.info("Compaction for {} end in PartialFlushIndexPhase.", metadata.getEntryLogId());
                return;
            }
            logRemovalListener.removeEntryLog(metadata.getEntryLogId());
            LOG.info("Compacted entry log : {}.", metadata.getEntryLogId());
        }

        synchronized void compactWithLogFlushFailure(EntryLogMetadata metadata) throws IOException {
            LOG.info("Compacting entry log {}", metadata.getEntryLogId());
            CompactionEntryLog compactionLog = entryLogger.newCompactionLog(metadata.getEntryLogId());

            CompactionPhase scanEntryLog = new ScanEntryLogPhase(metadata, compactionLog);
            if (!scanEntryLog.run()) {
                LOG.info("Compaction for {} end in ScanEntryLogPhase.", metadata.getEntryLogId());
                return;
            }
            CompactionPhase logFlushFailurePhase = new LogFlushFailurePhase(compactionLog);
            if (!logFlushFailurePhase.run()) {
                LOG.info("Compaction for {} end in FlushCompactionLogPhase.", metadata.getEntryLogId());
                return;
            }
            CompactionPhase updateIndex = new UpdateIndexPhase(compactionLog);
            if (!updateIndex.run()) {
                LOG.info("Compaction for entry log {} end in UpdateIndexPhase.", metadata.getEntryLogId());
                return;
            }
            logRemovalListener.removeEntryLog(metadata.getEntryLogId());
            LOG.info("Compacted entry log : {}.", metadata.getEntryLogId());
        }

        private class PartialFlushIndexPhase extends UpdateIndexPhase {

            public PartialFlushIndexPhase(CompactionEntryLog compactionLog) {
                super(compactionLog);
            }

            @Override
            void start() throws IOException {
                compactionLog.makeAvailable();
                assertTrue(offsets.size() > 1);
                // only flush index for one entry location
                EntryLocation el = offsets.get(0);
                ledgerStorage.updateEntriesLocations(offsets);
                ledgerStorage.flushEntriesLocationsIndex();
                throw new IOException("Flush ledger index encounter exception");
            }
        }

        private class LogFlushFailurePhase extends FlushCompactionLogPhase {

            LogFlushFailurePhase(CompactionEntryLog compactionEntryLog) {
                super(compactionEntryLog);
            }

            @Override
            void start() throws IOException {
                // flush the current compaction log
                compactionLog.flush();
                throw new IOException("Encounter IOException when trying to flush compaction log");
            }
        }
    }

}
