/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
#include "ConsumerImpl.h"

#include <pulsar/DeadLetterPolicyBuilder.h>
#include <pulsar/EncryptionContext.h>
#include <pulsar/MessageIdBuilder.h>

#include <algorithm>
#include <utility>
#include <variant>

#include "AckGroupingTracker.h"
#include "AckGroupingTrackerDisabled.h"
#include "AckGroupingTrackerEnabled.h"
#include "AsioDefines.h"
#include "BatchMessageAcker.h"
#include "BatchedMessageIdImpl.h"
#include "BitSet.h"
#include "ChunkMessageIdImpl.h"
#include "ClientConnection.h"
#include "ClientImpl.h"
#include "Commands.h"
#include "ExecutorService.h"
#include "GetLastMessageIdResponse.h"
#include "LogUtils.h"
#include "MessageCrypto.h"
#include "MessageIdUtil.h"
#include "MessageImpl.h"
#include "MessagesImpl.h"
#include "ProducerConfigurationImpl.h"
#include "PulsarApi.pb.h"
#include "ResultUtils.h"
#include "TimeUtils.h"
#include "TopicName.h"
#include "UnAckedMessageTrackerDisabled.h"
#include "UnAckedMessageTrackerEnabled.h"
#include "Utils.h"
#include "stats/ConsumerStatsDisabled.h"
#include "stats/ConsumerStatsImpl.h"

namespace pulsar {

DECLARE_LOG_OBJECT()

using std::chrono::milliseconds;
using std::chrono::seconds;

static optional<MessageId> getStartMessageId(const optional<MessageId>& startMessageId, bool inclusive) {
    if (!inclusive || !startMessageId) {
        return startMessageId;
    }
    // The default ledger id and entry id of a chunked message refer the fields of the last chunk. When the
    // start message id is inclusive, we need to start from the first chunk.
    auto chunkMsgIdImpl =
        dynamic_cast<const ChunkMessageIdImpl*>(Commands::getMessageIdImpl(startMessageId.value()).get());
    if (chunkMsgIdImpl) {
        return optional<MessageId>{chunkMsgIdImpl->getChunkedMessageIds().front()};
    }
    return startMessageId;
}

static std::shared_ptr<AckGroupingTracker> newAckGroupingTracker(const std::string& topic,
                                                                 const ConsumerConfiguration& config,
                                                                 const ClientImplPtr& client) {
    if (TopicName::get(topic)->isPersistent()) {
        if (config.getAckGroupingTimeMs() > 0) {
            return std::make_shared<AckGroupingTrackerEnabled>(
                config.getAckGroupingTimeMs(), config.getAckGroupingMaxSize(), config.isAckReceiptEnabled(),
                client->getIOExecutorProvider()->get());
        } else {
            return std::make_shared<AckGroupingTrackerDisabled>();
        }
    } else {
        return std::make_shared<AckGroupingTracker>();
    }
}

ConsumerImpl::ConsumerImpl(const ClientImplPtr& client, const std::string& topic,
                           const std::string& subscriptionName, const ConsumerConfiguration& conf,
                           bool isPersistent, const ConsumerInterceptorsPtr& interceptors,
                           const ExecutorServicePtr& listenerExecutor /* = NULL by default */,
                           bool hasParent /* = false by default */,
                           const ConsumerTopicType consumerTopicType /* = NonPartitioned by default */,
                           Commands::SubscriptionMode subscriptionMode,
                           const optional<MessageId>& startMessageId)
    : ConsumerImplBase(
          client, topic,
          Backoff(milliseconds(client->getClientConfig().getInitialBackoffIntervalMs()),
                  milliseconds(client->getClientConfig().getMaxBackoffIntervalMs()), milliseconds(0)),
          conf, listenerExecutor ? listenerExecutor : client->getListenerExecutorProvider()->get()),
      waitingForZeroQueueSizeMessage(false),
      config_(conf),
      subscription_(subscriptionName),
      originalSubscriptionName_(subscriptionName),
      isPersistent_(isPersistent),
      messageListener_(config_.getMessageListener()),
      eventListener_(config_.getConsumerEventListener()),
      hasParent_(hasParent),
      consumerTopicType_(consumerTopicType),
      subscriptionMode_(subscriptionMode),
      // This is the initial capacity of the queue
      incomingMessages_(std::max(config_.getReceiverQueueSize(), 1)),
      availablePermits_(0),
      receiverQueueRefillThreshold_(config_.getReceiverQueueSize() / 2),
      consumerId_(client->newConsumerId()),
      consumerStr_("[" + topic + ", " + subscriptionName + ", " + std::to_string(consumerId_) + "] "),
      messageListenerRunning_(!conf.isStartPaused()),
      negativeAcksTracker_(std::make_shared<NegativeAcksTracker>(client, *this, conf)),
      ackGroupingTrackerPtr_(newAckGroupingTracker(topic, conf, client)),
      readCompacted_(conf.isReadCompacted()),
      startMessageIdFromConfig_(pulsar::getStartMessageId(startMessageId, conf.isStartMessageIdInclusive())),
      startMessageId_(startMessageIdFromConfig_),
      maxPendingChunkedMessage_(conf.getMaxPendingChunkedMessage()),
      autoAckOldestChunkedMessageOnQueueFull_(conf.isAutoAckOldestChunkedMessageOnQueueFull()),
      expireTimeOfIncompleteChunkedMessageMs_(conf.getExpireTimeOfIncompleteChunkedMessageMs()),
      interceptors_(interceptors),
      requestIdGenerator_(client->getRequestIdGenerator()) {
    // Initialize un-ACKed messages OT tracker.
    if (conf.getUnAckedMessagesTimeoutMs() != 0) {
        if (conf.getTickDurationInMs() > 0) {
            unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerEnabled(
                conf.getUnAckedMessagesTimeoutMs(), conf.getTickDurationInMs(), client, *this));
        } else {
            unAckedMessageTrackerPtr_.reset(
                new UnAckedMessageTrackerEnabled(conf.getUnAckedMessagesTimeoutMs(), client, *this));
        }
    } else {
        unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled());
    }
    unAckedMessageTrackerPtr_->start();

    // Setup stats reporter.
    unsigned int statsIntervalInSeconds = client->getClientConfig().getStatsIntervalInSeconds();
    if (statsIntervalInSeconds) {
        consumerStatsBasePtr_ = std::make_shared<ConsumerStatsImpl>(
            consumerStr_, client->getIOExecutorProvider()->get()->createDeadlineTimer(),
            statsIntervalInSeconds);
    } else {
        consumerStatsBasePtr_ = std::make_shared<ConsumerStatsDisabled>();
    }
    consumerStatsBasePtr_->start();

    // Create msgCrypto
    if (conf.isEncryptionEnabled()) {
        msgCrypto_ = std::make_shared<MessageCrypto>(consumerStr_, false);
    }

    // Config dlq
    const auto& deadLetterPolicy = conf.getDeadLetterPolicy();
    if (deadLetterPolicy.getMaxRedeliverCount() > 0) {
        auto deadLetterPolicyBuilder =
            DeadLetterPolicyBuilder()
                .maxRedeliverCount(deadLetterPolicy.getMaxRedeliverCount())
                .initialSubscriptionName(deadLetterPolicy.getInitialSubscriptionName());
        if (deadLetterPolicy.getDeadLetterTopic().empty()) {
            deadLetterPolicyBuilder.deadLetterTopic(topic + "-" + subscriptionName + DLQ_GROUP_TOPIC_SUFFIX);
        } else {
            deadLetterPolicyBuilder.deadLetterTopic(deadLetterPolicy.getDeadLetterTopic());
        }
        deadLetterPolicy_ = deadLetterPolicyBuilder.build();
    }

    checkExpiredChunkedTimer_ = executor_->createDeadlineTimer();
}

ConsumerImpl::~ConsumerImpl() {
    auto client = client_.lock();
    if (state_ == Ready) {
        // this could happen at least in this condition:
        //      consumer seek, caused reconnection, if consumer close happened before connection ready,
        //      then consumer will not send closeConsumer to Broker side, and caused a leak of consumer in
        //      broker.
        if (client) {
            LOG_WARN(consumerStr_ << "Destroyed consumer which was not properly closed");
        }

        ClientConnectionPtr cnx = getCnx().lock();
        if (cnx) {
            auto requestId = newRequestId();
            cnx->sendRequestWithId(Commands::newCloseConsumer(consumerId_, requestId), requestId,
                                   "CLOSE_CONSUMER");
            cnx->removeConsumer(consumerId_);
        }
    }
    internalShutdown();
}

void ConsumerImpl::setPartitionIndex(int partitionIndex) { partitionIndex_ = partitionIndex; }

int ConsumerImpl::getPartitionIndex() { return partitionIndex_; }

Future<Result, ConsumerImplBaseWeakPtr> ConsumerImpl::getConsumerCreatedFuture() {
    return consumerCreatedPromise_.getFuture();
}

const std::string& ConsumerImpl::getSubscriptionName() const { return originalSubscriptionName_; }

const std::string& ConsumerImpl::getTopic() const { return topic(); }

void ConsumerImpl::start() {
    HandlerBase::start();
    ackGroupingTrackerPtr_->start(get_shared_this_ptr());
}

void ConsumerImpl::beforeConnectionChange(ClientConnection& cnx) { cnx.removeConsumer(consumerId_); }

void ConsumerImpl::onNegativeAcksSend(const std::set<MessageId>& messageIds) {
    interceptors_->onNegativeAcksSend(Consumer(shared_from_this()), messageIds);
}

Future<Error, bool> ConsumerImpl::connectionOpened(const ClientConnectionPtr& cnx) {
    Promise<Error, bool> promise;

    if (state_ == Closed) {
        LOG_DEBUG(getName() << "connectionOpened : Consumer is already closed");
        promise.setFailed(Error{ResultAlreadyClosed, ""});
        return promise.getFuture();
    }

    optional<MessageId> subscribeMessageId;
    {
        LockGuard lock{mutex_};
        setCnx(cnx);
        cnx->registerConsumer(consumerId_, get_shared_this_ptr());
        LOG_DEBUG(cnx->cnxString() << "Registered consumer " << consumerId_);

        clearReceiveQueue();
        subscribeMessageId =
            (subscriptionMode_ == Commands::SubscriptionModeNonDurable) ? startMessageId_ : std::nullopt;
        lastDequedMessageId_ = MessageId::earliest();
    }

    unAckedMessageTrackerPtr_->clear();

    ClientImplPtr client = client_.lock();
    auto requestId = newRequestId();
    SharedBuffer cmd = Commands::newSubscribe(
        topic(), subscription_, consumerId_, requestId, getSubType(), getConsumerName(), subscriptionMode_,
        subscribeMessageId, readCompacted_, config_.getProperties(), config_.getSubscriptionProperties(),
        config_.getSchema(), getInitialPosition(), config_.isReplicateSubscriptionStateEnabled(),
        config_.getKeySharedPolicy(), config_.getPriorityLevel());

    // Keep a reference to ensure object is kept alive.
    auto self = get_shared_this_ptr();
    setFirstRequestIdAfterConnect(requestId);
    cnx->sendRequestWithId(cmd, requestId, "SUBSCRIBE")
        .addListener([this, self, cnx, promise](const Error& error, const ResponseData& responseData) {
            Result handleResult = handleCreateConsumer(cnx, error.result);
            if (handleResult != ResultOk) {
                promise.setFailed(Error{handleResult, ""});
                return;
            }
            promise.setSuccess();
            // Complete the seek callback after completing `promise`, otherwise `reconnectionPending_` will
            // still be true when the seek operation is done.
            LockGuard lock{mutex_};
            if (seekStatus_ == SeekStatus::COMPLETED) {
                executor_->postWork([seekCallback{std::exchange(seekCallback_, std::nullopt).value()}]() {
                    seekCallback(ResultOk);
                });
                seekStatus_ = SeekStatus::NOT_STARTED;
            }
        });

    return promise.getFuture();
}

void ConsumerImpl::connectionFailed(Result result) {
    // Keep a reference to ensure object is kept alive
    auto ptr = get_shared_this_ptr();

    if (!isResultRetryable(result) && consumerCreatedPromise_.setFailed(result)) {
        state_ = Failed;
    }
}

void ConsumerImpl::sendFlowPermitsToBroker(const ClientConnectionPtr& cnx, int numMessages) {
    if (cnx && numMessages > 0) {
        LOG_DEBUG(getName() << "Send more permits: " << numMessages);
        SharedBuffer cmd = Commands::newFlow(consumerId_, static_cast<unsigned int>(numMessages));
        cnx->sendCommand(cmd);
    }
}

Result ConsumerImpl::handleCreateConsumer(const ClientConnectionPtr& cnx, Result result) {
    Result handleResult = ResultOk;

    if (result == ResultOk) {
        {
            Lock mutexLock(mutex_);
            if (!changeToReadyState()) {
                auto client = client_.lock();
                if (client) {
                    LOG_INFO(getName() << "Closing subscribed consumer since it was already closed");
                    int requestId = client->newRequestId();
                    auto name = getName();
                    cnx->sendRequestWithId(Commands::newCloseConsumer(consumerId_, requestId), requestId,
                                           "CLOSE_CONSUMER")
                        .addListener([name](const Error& error, const ResponseData&) {
                            if (error.result == ResultOk) {
                                LOG_INFO(name << "Closed consumer successfully after subscribe completed");
                            } else {
                                LOG_WARN(name << "Failed to close consumer: " << strResult(error.result));
                            }
                        });
                } else {
                    // This should not happen normally because if client is destroyed, the connection pool
                    // should also be closed, which means all connections should be closed. Close the
                    // connection to let broker know this registered consumer is inactive.
                    LOG_WARN(getName()
                             << "Client already closed when subscribe completed, close the connection "
                             << cnx->cnxString());
                    cnx->close(Error{ResultNotConnected, ""});
                }
                return ResultAlreadyClosed;
            }

            mutexLock.unlock();
            LOG_INFO(getName() << "Created consumer on broker " << cnx->cnxString());
            incomingMessages_.clear();
            possibleSendToDeadLetterTopicMessages_.clear();
            backoff_.reset();
            if (!messageListener_ && config_.getReceiverQueueSize() == 0) {
                // Complicated logic since we don't have a isLocked() function for mutex
                if (waitingForZeroQueueSizeMessage) {
                    sendFlowPermitsToBroker(cnx, 1);
                }
                // Note that the order of lock acquisition must be mutex_ -> pendingReceiveMutex_,
                // otherwise a deadlock will occur.
                Lock pendingReceiveMutexLock(pendingReceiveMutex_);
                if (!pendingReceives_.empty()) {
                    sendFlowPermitsToBroker(cnx, pendingReceives_.size());
                }
            }
            availablePermits_ = 0;
        }

        LOG_DEBUG(getName() << "Send initial flow permits: " << config_.getReceiverQueueSize());
        if (config_.getReceiverQueueSize() != 0) {
            sendFlowPermitsToBroker(cnx, config_.getReceiverQueueSize());
        } else if (messageListener_) {
            sendFlowPermitsToBroker(cnx, 1);
        }
        consumerCreatedPromise_.setValue(get_shared_this_ptr());
    } else {
        if (result == ResultTimeout) {
            // Creating the consumer has timed out. We need to ensure the broker closes the consumer
            // in case it was indeed created, otherwise it might prevent new subscribe operation,
            // since we are not closing the connection
            auto requestId = newRequestId();
            cnx->sendRequestWithId(Commands::newCloseConsumer(consumerId_, requestId), requestId,
                                   "CLOSE_CONSUMER");
        }

        if (consumerCreatedPromise_.isComplete()) {
            // Clear the connection set before SUBSCRIBE so the next reconnect is not skipped.
            LOG_WARN(getName() << "Failed to reconnect consumer: " << strResult(result));
            resetCnx();
            handleResult = ResultRetryable;
        } else {
            // Consumer was not yet created, retry to connect to broker if it's possible
            handleResult = convertToTimeoutIfNecessary(result, creationTimestamp_);
            if (isResultRetryable(handleResult)) {
                LOG_WARN(getName() << "Temporary error in creating consumer: " << strResult(handleResult));
            } else {
                LOG_ERROR(getName() << "Failed to create consumer: " << strResult(handleResult));
                consumerCreatedPromise_.setFailed(handleResult);
                state_ = Failed;
            }
        }
    }

    return handleResult;
}

void ConsumerImpl::unsubscribeAsync(const ResultCallback& originalCallback) {
    LOG_INFO(getName() << "Unsubscribing");

    auto callback = [this, originalCallback](Result result) {
        if (result == ResultOk) {
            internalShutdown();
            LOG_INFO(getName() << "Unsubscribed successfully");
        } else {
            state_ = Ready;
            LOG_WARN(getName() << "Failed to unsubscribe: " << result);
        }
        if (originalCallback) {
            originalCallback(result);
        }
    };

    if (state_ != Ready) {
        callback(ResultAlreadyClosed);
        return;
    }

    Lock lock(mutex_);

    ClientConnectionPtr cnx = getCnx().lock();
    if (cnx) {
        LOG_DEBUG(getName() << "Unsubscribe request sent for consumer - " << consumerId_);
        ClientImplPtr client = client_.lock();
        lock.unlock();
        auto requestId = newRequestId();
        SharedBuffer cmd = Commands::newUnsubscribe(consumerId_, requestId);
        auto self = get_shared_this_ptr();
        cnx->sendRequestWithId(cmd, requestId, "UNSUBSCRIBE")
            .addListener(
                [self, callback](const Error& error, const ResponseData&) { callback(error.result); });
    } else {
        Result result = ResultNotConnected;
        lock.unlock();
        LOG_WARN(getName() << "Failed to unsubscribe: " << strResult(result));
        callback(result);
    }
}

void ConsumerImpl::discardChunkMessages(const std::string& uuid, const MessageId& messageId, bool autoAck) {
    if (autoAck) {
        acknowledgeAsync(messageId, [uuid, messageId](Result result) {
            if (result != ResultOk) {
                LOG_WARN("Failed to acknowledge discarded chunk, uuid: " << uuid
                                                                         << ", messageId: " << messageId);
            }
        });
    } else {
        trackMessage(messageId);
    }
}

void ConsumerImpl::triggerCheckExpiredChunkedTimer() {
    checkExpiredChunkedTimer_->expires_after(milliseconds(expireTimeOfIncompleteChunkedMessageMs_));
    auto weakSelf = weak_from_this();
    checkExpiredChunkedTimer_->async_wait([this, weakSelf](const ASIO_ERROR& ec) -> void {
        auto self = weakSelf.lock();
        if (!self) {
            return;
        }
        if (ec) {
            LOG_DEBUG(getName() << " Check expired chunked messages was failed or cancelled, code[" << ec
                                << "].");
            return;
        }
        Lock lock(chunkProcessMutex_);
        long currentTimeMs = TimeUtils::currentTimeMillis();
        chunkedMessageCache_.removeOldestValuesIf(
            [this, currentTimeMs](const std::string& uuid, const ChunkedMessageCtx& ctx) -> bool {
                bool expired =
                    currentTimeMs > ctx.getReceivedTimeMs() + expireTimeOfIncompleteChunkedMessageMs_;
                if (!expired) {
                    return false;
                }
                for (const MessageId& msgId : ctx.getChunkedMessageIds()) {
                    LOG_INFO("Removing expired chunk messages: uuid: " << uuid << ", messageId: " << msgId);
                    discardChunkMessages(uuid, msgId, true);
                }
                return true;
            });
        triggerCheckExpiredChunkedTimer();
        return;
    });
}

optional<SharedBuffer> ConsumerImpl::processMessageChunk(const SharedBuffer& payload,
                                                         const proto::MessageMetadata& metadata,
                                                         const proto::MessageIdData& messageIdData,
                                                         const ClientConnectionPtr& cnx,
                                                         MessageId& messageId) {
    const auto chunkId = metadata.chunk_id();
    const auto& uuid = metadata.uuid();
    LOG_DEBUG("Process message chunk (chunkId: " << chunkId << ", uuid: " << uuid
                                                 << ", messageId: " << messageId << ") of "
                                                 << payload.readableBytes() << " bytes");

    Lock lock(chunkProcessMutex_);

    // Lazy task scheduling to expire incomplete chunk message
    bool expected = false;
    if (expireTimeOfIncompleteChunkedMessageMs_ > 0 &&
        expireChunkMessageTaskScheduled_.compare_exchange_strong(expected, true)) {
        triggerCheckExpiredChunkedTimer();
    }

    auto it = chunkedMessageCache_.find(uuid);

    if (chunkId == 0 && it == chunkedMessageCache_.end()) {
        if (maxPendingChunkedMessage_ > 0 && chunkedMessageCache_.size() >= maxPendingChunkedMessage_) {
            chunkedMessageCache_.removeOldestValues(
                chunkedMessageCache_.size() - maxPendingChunkedMessage_ + 1,
                [this](const std::string& uuid, const ChunkedMessageCtx& ctx) {
                    for (const MessageId& msgId : ctx.getChunkedMessageIds()) {
                        discardChunkMessages(uuid, msgId, autoAckOldestChunkedMessageOnQueueFull_);
                    }
                });
        }
        it = chunkedMessageCache_.putIfAbsent(
            uuid, ChunkedMessageCtx{metadata.num_chunks_from_msg(), metadata.total_chunk_msg_size()});
    }

    auto& chunkedMsgCtx = it->second;
    if (it == chunkedMessageCache_.end() || !chunkedMsgCtx.validateChunkId(chunkId)) {
        auto startMessageId = getStartMessageId();
        if (!config_.isStartMessageIdInclusive() && startMessageId &&
            startMessageId->ledgerId() == messageId.ledgerId() &&
            startMessageId->entryId() == messageId.entryId()) {
            // When the start message id is not inclusive, the last chunk of the previous chunked message will
            // be delivered, which is expected and we only need to filter it out.
            chunkedMessageCache_.remove(uuid);
            LOG_INFO("Filtered the chunked message before the start message id (uuid: "
                     << uuid << " chunkId: " << chunkId << ", messageId: " << messageId << ")");
        } else if (it == chunkedMessageCache_.end()) {
            LOG_ERROR("Received an uncached chunk (uuid: " << uuid << " chunkId: " << chunkId
                                                           << ", messageId: " << messageId << ")");
        } else {
            LOG_ERROR("Received a chunk whose chunk id is invalid (uuid: "
                      << uuid << " chunkId: " << chunkId << ", messageId: " << messageId << ")");
            chunkedMessageCache_.remove(uuid);
        }
        lock.unlock();
        increaseAvailablePermits(cnx);
        trackMessage(messageId);
        return {};
    }

    chunkedMsgCtx.appendChunk(messageId, payload);
    if (!chunkedMsgCtx.isCompleted()) {
        lock.unlock();
        increaseAvailablePermits(cnx);
        return {};
    }

    messageId = std::make_shared<ChunkMessageIdImpl>(chunkedMsgCtx.moveChunkedMessageIds())->build();

    LOG_DEBUG("Chunked message completed chunkId: " << chunkId << ", ChunkedMessageCtx: " << chunkedMsgCtx
                                                    << ", sequenceId: " << metadata.sequence_id());

    auto wholePayload = chunkedMsgCtx.getBuffer();
    chunkedMessageCache_.remove(uuid);
    if (uncompressMessageIfNeeded(cnx, messageIdData, metadata, wholePayload, false)) {
        return wholePayload;
    } else {
        return {};
    }
}

void ConsumerImpl::messageReceived(const ClientConnectionPtr& cnx, const proto::CommandMessage& msg,
                                   bool& isChecksumValid, proto::BrokerEntryMetadata& brokerEntryMetadata,
                                   proto::MessageMetadata& metadata, SharedBuffer& payload) {
    LOG_DEBUG(getName() << "Received Message -- Size: " << payload.readableBytes());

    if (!isChecksumValid) {
        // Message discarded for checksum error
        discardCorruptedMessage(cnx, msg.message_id(), CommandAck_ValidationError_ChecksumMismatch);
        return;
    }

    auto encryptionContext = metadata.encryption_keys_size() > 0
                                 ? optional<EncryptionContext>(std::in_place, metadata, false)
                                 : std::nullopt;
    const auto decryptionResult = decryptMessageIfNeeded(cnx, msg, encryptionContext, payload);
    if (decryptionResult == DecryptionResult::FAILED) {
        // Message was discarded or not consumed due to decryption failure
        return;
    } else if (decryptionResult == DecryptionResult::CONSUME_ENCRYPTED && encryptionContext.has_value()) {
        // Message is encrypted, but we let the application consume it as-is
        encryptionContext->setDecryptionFailed(true);
    }

    auto redeliveryCount = msg.redelivery_count();
    const bool isChunkedMessage = metadata.num_chunks_from_msg() > 1;
    if (decryptionResult == DecryptionResult::SUCCESS && !isChunkedMessage) {
        if (!uncompressMessageIfNeeded(cnx, msg.message_id(), metadata, payload, true)) {
            // Message was discarded on decompression error
            return;
        }
    }

    const auto& messageIdData = msg.message_id();
    auto messageId = MessageIdBuilder::from(messageIdData).batchIndex(-1).build();

    // Only a non-batched messages can be a chunk
    if (!metadata.has_num_messages_in_batch() && isChunkedMessage) {
        auto optionalPayload = processMessageChunk(payload, metadata, messageIdData, cnx, messageId);
        if (optionalPayload) {
            payload = optionalPayload.value();
        } else {
            return;
        }
    }

    Message m(messageId, brokerEntryMetadata, metadata, payload);
    m.impl_->cnx_ = cnx.get();
    m.impl_->setTopicName(getTopicPtr());
    m.impl_->setRedeliveryCount(msg.redelivery_count());
    m.impl_->encryptionContext_ = std::move(encryptionContext);

    if (metadata.has_schema_version()) {
        m.impl_->setSchemaVersion(metadata.schema_version());
    }

    LOG_DEBUG(getName() << " metadata.num_messages_in_batch() = " << metadata.num_messages_in_batch());
    LOG_DEBUG(getName() << " metadata.has_num_messages_in_batch() = "
                        << metadata.has_num_messages_in_batch());

    const auto state = state_.load(std::memory_order_relaxed);
    if (state == Closing || state == Closed) {
        return;
    }
    if (!listenerExecutor_) {
        LOG_ERROR(getName() << " listenerExecutor_ is null, discarding message to avoid null dereference");
        increaseAvailablePermits(cnx);
        return;
    }
    uint32_t numOfMessageReceived = m.impl_->metadata.num_messages_in_batch();
    if (ackGroupingTrackerPtr_->isDuplicate(m.getMessageId())) {
        LOG_DEBUG(getName() << " Ignoring message as it was ACKed earlier by same consumer.");
        increaseAvailablePermits(cnx, numOfMessageReceived);
        return;
    }

    if (metadata.has_num_messages_in_batch() && decryptionResult == DecryptionResult::SUCCESS) {
        BitSet::Data words(msg.ack_set_size());
        for (int i = 0; i < words.size(); i++) {
            words[i] = msg.ack_set(i);
        }
        BitSet ackSet{std::move(words)};
        numOfMessageReceived = receiveIndividualMessagesFromBatch(cnx, m, ackSet, msg.redelivery_count());
    } else {
        // try convert key value data.
        m.impl_->convertPayloadToKeyValue(config_.getSchema());

        const auto startMessageId = getStartMessageId();
        if (isPersistent_ && startMessageId && m.getMessageId().ledgerId() == startMessageId->ledgerId() &&
            isPrior(m.getMessageId().entryId(), startMessageId->entryId())) {
            LOG_DEBUG(getName() << " Ignoring message from before the startMessageId: "
                                << startMessageId.value());
            return;
        }
        if (redeliveryCount >= deadLetterPolicy_.getMaxRedeliverCount()) {
            possibleSendToDeadLetterTopicMessages_.put(m.getMessageId(), std::vector<Message>{m});
            if (redeliveryCount > deadLetterPolicy_.getMaxRedeliverCount()) {
                redeliverUnacknowledgedMessages({m.getMessageId()});
                increaseAvailablePermits(cnx);
                return;
            }
        }
        executeNotifyCallback(m);
    }

    if (messageListener_) {
        if (!messageListenerRunning_) {
            return;
        }
        // Trigger message listener callback in a separate thread
        if (listenerExecutor_) {
            while (numOfMessageReceived--) {
                listenerExecutor_->postWork(
                    std::bind(&ConsumerImpl::internalListener, get_shared_this_ptr()));
            }
        }
    }
}

void ConsumerImpl::activeConsumerChanged(bool isActive) {
    if (eventListener_) {
        listenerExecutor_->postWork(
            std::bind(&ConsumerImpl::internalConsumerChangeListener, get_shared_this_ptr(), isActive));
    }
}

void ConsumerImpl::internalConsumerChangeListener(bool isActive) {
    try {
        if (isActive) {
            eventListener_->becameActive(Consumer(get_shared_this_ptr()), partitionIndex_);
        } else {
            eventListener_->becameInactive(Consumer(get_shared_this_ptr()), partitionIndex_);
        }
    } catch (const std::exception& e) {
        LOG_ERROR(getName() << "Exception thrown from event listener " << e.what());
    }
}

void ConsumerImpl::failPendingReceiveCallback() {
    Message msg;
    Lock lock(pendingReceiveMutex_);
    while (!pendingReceives_.empty()) {
        ReceiveCallback callback = pendingReceives_.front();
        pendingReceives_.pop();
        listenerExecutor_->postWork(std::bind(&ConsumerImpl::notifyPendingReceivedCallback,
                                              get_shared_this_ptr(), ResultAlreadyClosed, msg, callback));
    }
    lock.unlock();
}

void ConsumerImpl::executeNotifyCallback(Message& msg) {
    Lock lock(pendingReceiveMutex_);
    // if asyncReceive is waiting then notify callback without adding to incomingMessages queue
    bool asyncReceivedWaiting = !pendingReceives_.empty();
    ReceiveCallback callback;
    if (asyncReceivedWaiting) {
        callback = pendingReceives_.front();
        pendingReceives_.pop();
    }
    lock.unlock();

    // has pending receive, direct callback.
    if (asyncReceivedWaiting) {
        if (listenerExecutor_) {
            listenerExecutor_->postWork(std::bind(&ConsumerImpl::notifyPendingReceivedCallback,
                                                  get_shared_this_ptr(), ResultOk, msg, callback));
        } else {
            notifyPendingReceivedCallback(ResultOk, msg, callback);
        }
        return;
    }

    // try to add incoming messages.
    // config_.getReceiverQueueSize() != 0 or waiting For ZeroQueueSize Message`
    if (messageListener_ || config_.getReceiverQueueSize() != 0 || waitingForZeroQueueSizeMessage) {
        incomingMessages_.push(msg);
        incomingMessagesSize_.fetch_add(msg.getLength());
    }

    // try trigger pending batch messages
    Lock batchOptionLock(batchReceiveOptionMutex_);
    if (hasEnoughMessagesForBatchReceive()) {
        ConsumerImplBase::notifyBatchPendingReceivedCallback();
    }
}

void ConsumerImpl::notifyBatchPendingReceivedCallback(const BatchReceiveCallback& callback) {
    auto messages = std::make_shared<MessagesImpl>(batchReceivePolicy_.getMaxNumMessages(),
                                                   batchReceivePolicy_.getMaxNumBytes());
    Message msg;
    while (incomingMessages_.popIf(
        msg, [&messages](const Message& peekMsg) { return messages->canAdd(peekMsg); })) {
        messageProcessed(msg);
        Message interceptMsg = interceptors_->beforeConsume(Consumer(shared_from_this()), msg);
        messages->add(interceptMsg);
    }
    auto self = get_shared_this_ptr();
    listenerExecutor_->postWork(
        [callback, messages, self]() { callback(ResultOk, messages->getMessageList()); });
}

void ConsumerImpl::notifyPendingReceivedCallback(Result result, Message& msg,
                                                 const ReceiveCallback& callback) {
    if (result == ResultOk && config_.getReceiverQueueSize() != 0) {
        messageProcessed(msg);
        msg = interceptors_->beforeConsume(Consumer(shared_from_this()), msg);
        unAckedMessageTrackerPtr_->add(msg.getMessageId());
    }
    callback(result, msg);
}

// Zero Queue size is not supported with Batch Messages
uint32_t ConsumerImpl::receiveIndividualMessagesFromBatch(const ClientConnectionPtr& cnx,
                                                          Message& batchedMessage, const BitSet& ackSet,
                                                          int redeliveryCount) {
    auto batchSize = batchedMessage.impl_->metadata.num_messages_in_batch();
    LOG_DEBUG("Received Batch messages of size - " << batchSize
                                                   << " -- msgId: " << batchedMessage.getMessageId());
    const auto startMessageId = getStartMessageId();

    int skippedMessages = 0;

    auto acker = BatchMessageAckerImpl::create(batchSize);
    std::vector<Message> possibleToDeadLetter;
    for (int i = 0; i < batchSize; i++) {
        // This is a cheap copy since message contains only one shared pointer (impl_)
        Message msg = Commands::deSerializeSingleMessageInBatch(batchedMessage, i, batchSize, acker);
        msg.impl_->setRedeliveryCount(redeliveryCount);
        msg.impl_->setTopicName(batchedMessage.impl_->topicName_);
        msg.impl_->convertPayloadToKeyValue(config_.getSchema());
        if (msg.impl_->brokerEntryMetadata.has_index()) {
            msg.impl_->brokerEntryMetadata.set_index(msg.impl_->brokerEntryMetadata.index() - batchSize + i +
                                                     1);
        }

        if (redeliveryCount >= deadLetterPolicy_.getMaxRedeliverCount()) {
            possibleToDeadLetter.emplace_back(msg);
            if (redeliveryCount > deadLetterPolicy_.getMaxRedeliverCount()) {
                skippedMessages++;
                continue;
            }
        }

        if (startMessageId) {
            const MessageId& msgId = msg.getMessageId();

            // If we are receiving a batch message, we need to discard messages that were prior
            // to the startMessageId
            if (isPersistent_ && msgId.ledgerId() == startMessageId->ledgerId() &&
                msgId.entryId() == startMessageId->entryId() &&
                isPrior(msgId.batchIndex(), startMessageId->batchIndex())) {
                LOG_DEBUG(getName() << "Ignoring message from before the startMessageId"
                                    << msg.getMessageId());
                ++skippedMessages;
                continue;
            }
        }

        if (!ackSet.isEmpty() && !ackSet.get(i)) {
            LOG_DEBUG(getName() << "Ignoring message from " << i
                                << "th message, which has been acknowledged");
            ++skippedMessages;
            continue;
        }

        executeNotifyCallback(msg);
    }

    if (!possibleToDeadLetter.empty()) {
        possibleSendToDeadLetterTopicMessages_.put(batchedMessage.getMessageId(), possibleToDeadLetter);
        if (redeliveryCount > deadLetterPolicy_.getMaxRedeliverCount()) {
            redeliverUnacknowledgedMessages({batchedMessage.getMessageId()});
        }
    }

    if (skippedMessages > 0) {
        increaseAvailablePermits(cnx, skippedMessages);
    }

    return batchSize - skippedMessages;
}

auto ConsumerImpl::decryptMessageIfNeeded(const ClientConnectionPtr& cnx, const proto::CommandMessage& msg,
                                          const optional<EncryptionContext>& context, SharedBuffer& payload)
    -> DecryptionResult {
    if (!context.has_value()) {
        return DecryptionResult::SUCCESS;
    }

    // If KeyReader is not configured throw exception based on config param
    if (!config_.isEncryptionEnabled()) {
        if (config_.getCryptoFailureAction() == ConsumerCryptoFailureAction::CONSUME) {
            LOG_WARN(getName() << "CryptoKeyReader is not implemented. Consuming encrypted message.");
            return DecryptionResult::CONSUME_ENCRYPTED;
        } else if (config_.getCryptoFailureAction() == ConsumerCryptoFailureAction::DISCARD) {
            LOG_WARN(getName() << "Skipping decryption since CryptoKeyReader is not implemented and config "
                                  "is set to discard");
            discardCorruptedMessage(cnx, msg.message_id(), CommandAck_ValidationError_DecryptionError);
        } else {
            LOG_ERROR(getName() << "Message delivery failed since CryptoKeyReader is not implemented to "
                                   "consume encrypted message");
            auto messageId = MessageIdBuilder::from(msg.message_id()).build();
            unAckedMessageTrackerPtr_->add(messageId);
        }
        return DecryptionResult::FAILED;
    }

    SharedBuffer decryptedPayload;
    if (msgCrypto_->decrypt(*context, payload, config_.getCryptoKeyReader(), decryptedPayload)) {
        payload = decryptedPayload;
        return DecryptionResult::SUCCESS;
    }

    if (config_.getCryptoFailureAction() == ConsumerCryptoFailureAction::CONSUME) {
        // Note, batch message will fail to consume even if config is set to consume
        LOG_WARN(
            getName() << "Decryption failed. Consuming encrypted message since config is set to consume.");
        return DecryptionResult::CONSUME_ENCRYPTED;
    } else if (config_.getCryptoFailureAction() == ConsumerCryptoFailureAction::DISCARD) {
        LOG_WARN(getName() << "Discarding message since decryption failed and config is set to discard");
        discardCorruptedMessage(cnx, msg.message_id(), CommandAck_ValidationError_DecryptionError);
    } else {
        LOG_ERROR(getName() << "Message delivery failed since unable to decrypt incoming message");
        auto messageId = MessageIdBuilder::from(msg.message_id()).build();
        unAckedMessageTrackerPtr_->add(messageId);
    }
    return DecryptionResult::FAILED;
}

bool ConsumerImpl::uncompressMessageIfNeeded(const ClientConnectionPtr& cnx,
                                             const proto::MessageIdData& messageIdData,
                                             const proto::MessageMetadata& metadata, SharedBuffer& payload,
                                             bool checkMaxMessageSize) {
    if (!metadata.has_compression()) {
        return true;
    }

    CompressionType compressionType = static_cast<CompressionType>(metadata.compression());

    uint32_t uncompressedSize = metadata.uncompressed_size();
    uint32_t payloadSize = payload.readableBytes();
    if (cnx) {
        if (checkMaxMessageSize && payloadSize > ClientConnection::getMaxMessageSize()) {
            // Uncompressed size is itself corrupted since it cannot be bigger than the MaxMessageSize
            LOG_ERROR(getName() << "Got corrupted payload message size " << payloadSize  //
                                << " at  " << messageIdData.ledgerid() << ":" << messageIdData.entryid());
            discardCorruptedMessage(cnx, messageIdData,
                                    CommandAck_ValidationError_UncompressedSizeCorruption);
            return false;
        }
    } else {
        LOG_ERROR("Connection not ready for Consumer - " << getConsumerId());
        return false;
    }

    if (!CompressionCodecProvider::getCodec(compressionType).decode(payload, uncompressedSize, payload)) {
        LOG_ERROR(getName() << "Failed to decompress message with " << uncompressedSize  //
                            << " at  " << messageIdData.ledgerid() << ":" << messageIdData.entryid());
        discardCorruptedMessage(cnx, messageIdData, CommandAck_ValidationError_DecompressionError);
        return false;
    }

    return true;
}

void ConsumerImpl::discardCorruptedMessage(const ClientConnectionPtr& cnx,
                                           const proto::MessageIdData& messageId,
                                           CommandAck_ValidationError validationError) {
    LOG_ERROR(getName() << "Discarding corrupted message at " << messageId.ledgerid() << ":"
                        << messageId.entryid());

    SharedBuffer cmd = Commands::newAck(consumerId_, messageId.ledgerid(), messageId.entryid(), {},
                                        CommandAck_AckType_Individual, validationError);

    cnx->sendCommand(cmd);
    increaseAvailablePermits(cnx);
}

void ConsumerImpl::internalListener() {
    if (!messageListenerRunning_) {
        return;
    }
    Message msg;
    if (!incomingMessages_.pop(msg, std::chrono::milliseconds(0))) {
        // This will only happen when the connection got reset and we cleared the queue
        return;
    }
    trackMessage(msg.getMessageId());
    try {
        consumerStatsBasePtr_->receivedMessage(msg, ResultOk);
        setLastDequedMessageId(msg.getMessageId());
        Consumer consumer{get_shared_this_ptr()};
        Message interceptMsg = interceptors_->beforeConsume(Consumer(shared_from_this()), msg);
        messageListener_(consumer, interceptMsg);
    } catch (const std::exception& e) {
        LOG_ERROR(getName() << "Exception thrown from listener" << e.what());
    }
    messageProcessed(msg, false);
}

Result ConsumerImpl::fetchSingleMessageFromBroker(Message& msg) {
    if (config_.getReceiverQueueSize() != 0) {
        LOG_ERROR(getName() << " Can't use receiveForZeroQueueSize if the queue size is not 0");
        return ResultInvalidConfiguration;
    }

    // Using RAII for locking
    Lock lock(mutexForReceiveWithZeroQueueSize);

    // Just being cautious
    if (incomingMessages_.size() != 0) {
        LOG_ERROR(
            getName() << "The incoming message queue should never be greater than 0 when Queue size is 0");
        incomingMessages_.clear();
    }

    {
        // Lock mutex_ to prevent a race condition with handleCreateConsumer.
        // If handleCreateConsumer is executed after setting waitingForZeroQueueSizeMessage to true and
        // before calling sendFlowPermitsToBroker, the result may be that a flow permit is sent twice.
        Lock lock(mutex_);
        waitingForZeroQueueSizeMessage = true;
        // If connection_ is nullptr, sendFlowPermitsToBroker does nothing.
        // In other words, a flow permit will not be sent until setCnx(cnx) is executed in
        // handleCreateConsumer.
        sendFlowPermitsToBroker(getCnx().lock(), 1);
    }

    while (true) {
        if (!incomingMessages_.pop(msg)) {
            return ResultInterrupted;
        }

        {
            // Lock needed to prevent race between connectionOpened and the check "msg.impl_->cnx_ ==
            // currentCnx.get())"
            Lock localLock(mutex_);
            // if message received due to an old flow - discard it and wait for the message from the
            // latest flow command
            ClientConnectionPtr currentCnx = getCnx().lock();
            if (msg.impl_->cnx_ == currentCnx.get()) {
                waitingForZeroQueueSizeMessage = false;
                // Can't use break here else it may trigger a race with connection opened.

                localLock.unlock();
                msg = interceptors_->beforeConsume(Consumer(shared_from_this()), msg);
                return ResultOk;
            }
        }
    }
}

Result ConsumerImpl::receive(Message& msg) {
    Result res = receiveHelper(msg);
    consumerStatsBasePtr_->receivedMessage(msg, res);
    return res;
}

void ConsumerImpl::receiveAsync(const ReceiveCallback& callback) {
    Message msg;

    // fail the callback if consumer is closing or closed
    if (state_ != Ready) {
        callback(ResultAlreadyClosed, msg);
        return;
    }

    if (messageListener_) {
        LOG_ERROR(getName() << "Can not receive when a listener has been set");
        callback(ResultInvalidConfiguration, msg);
        return;
    }

    Lock mutexlock(mutex_, std::defer_lock);
    if (config_.getReceiverQueueSize() == 0) {
        // Lock mutex_ to prevent a race condition with handleCreateConsumer.
        // If handleCreateConsumer is executed after pushing the callback to pendingReceives_ and
        // before calling sendFlowPermitsToBroker, the result may be that a flow permit is sent twice.
        // Note that the order of lock acquisition must be mutex_ -> pendingReceiveMutex_,
        // otherwise a deadlock will occur.
        mutexlock.lock();
    }

    Lock pendingReceiveMutexLock(pendingReceiveMutex_);
    if (incomingMessages_.pop(msg, std::chrono::milliseconds(0))) {
        pendingReceiveMutexLock.unlock();
        if (config_.getReceiverQueueSize() == 0) {
            mutexlock.unlock();
        }
        messageProcessed(msg);
        msg = interceptors_->beforeConsume(Consumer(shared_from_this()), msg);
        callback(ResultOk, msg);
    } else if (config_.getReceiverQueueSize() == 0) {
        pendingReceives_.push(callback);
        // If connection_ is nullptr, sendFlowPermitsToBroker does nothing.
        // In other words, a flow permit will not be sent until setCnx(cnx) is executed in
        // handleCreateConsumer.
        sendFlowPermitsToBroker(getCnx().lock(), 1);
        pendingReceiveMutexLock.unlock();
        mutexlock.unlock();
    } else {
        pendingReceives_.push(callback);
        pendingReceiveMutexLock.unlock();
    }
}

Result ConsumerImpl::receiveHelper(Message& msg) {
    if (state_ != Ready) {
        return ResultAlreadyClosed;
    }

    if (messageListener_) {
        LOG_ERROR(getName() << "Can not receive when a listener has been set");
        return ResultInvalidConfiguration;
    }

    if (config_.getReceiverQueueSize() == 0) {
        return fetchSingleMessageFromBroker(msg);
    }

    if (!incomingMessages_.pop(msg)) {
        return ResultInterrupted;
    }

    messageProcessed(msg);
    msg = interceptors_->beforeConsume(Consumer(shared_from_this()), msg);
    return ResultOk;
}

Result ConsumerImpl::receive(Message& msg, int timeout) {
    Result res = receiveHelper(msg, timeout);
    consumerStatsBasePtr_->receivedMessage(msg, res);
    return res;
}

Result ConsumerImpl::receiveHelper(Message& msg, int timeout) {
    if (config_.getReceiverQueueSize() == 0) {
        LOG_WARN(getName() << "Can't use this function if the queue size is 0");
        return ResultInvalidConfiguration;
    }

    if (state_ != Ready) {
        return ResultAlreadyClosed;
    }

    if (messageListener_) {
        LOG_ERROR(getName() << "Can not receive when a listener has been set");
        return ResultInvalidConfiguration;
    }

    if (incomingMessages_.pop(msg, std::chrono::milliseconds(timeout))) {
        messageProcessed(msg);
        msg = interceptors_->beforeConsume(Consumer(shared_from_this()), msg);
        return ResultOk;
    } else {
        if (state_ != Ready) {
            return ResultAlreadyClosed;
        }
        return ResultTimeout;
    }
}

void ConsumerImpl::messageProcessed(Message& msg, bool track) {
    setLastDequedMessageId(msg.getMessageId());
    incomingMessagesSize_.fetch_sub(msg.getLength());

    ClientConnectionPtr currentCnx = getCnx().lock();
    if (currentCnx && msg.impl_->cnx_ != currentCnx.get()) {
        LOG_DEBUG(getName() << "Not adding permit since connection is different.");
        return;
    }

    if (!hasParent_) {
        increaseAvailablePermits(currentCnx);
    }
    if (track) {
        trackMessage(msg.getMessageId());
    }
}

void ConsumerImpl::onClusterSwitching() {
    {
        LockGuard lock{mutex_};
        incomingMessages_.clear();
        startMessageId_ = startMessageIdFromConfig_;
        lastDequedMessageId_ = MessageId::earliest();
        lastMessageIdInBroker_ = MessageId::earliest();
        seekStatus_ = SeekStatus::NOT_STARTED;
        lastSeekArg_.reset();
    }
    setRedirectedClusterURI("");
    ackGroupingTrackerPtr_->flushAndClean();
}

/**
 * Clear the internal receiver queue and returns the message id of what was the 1st message in the queue that
 * was
 * not seen by the application
 * `startMessageId_` is updated so that we can discard messages after delivery restarts.
 * NOTE: `mutex_` must be locked before calling this method.
 */
void ConsumerImpl::clearReceiveQueue() {
    if (seekStatus_ != SeekStatus::NOT_STARTED) {
        // Flush the pending ACKs in case newly arrived messages are filtered out by the previous pending ACKs
        ackGroupingTrackerPtr_->flushAndClean();
        if (lastSeekArg_.has_value()) {
            if (std::holds_alternative<MessageId>(lastSeekArg_.value())) {
                startMessageId_ = std::get<MessageId>(lastSeekArg_.value());
            } else {
                // Invalidate startMessageId_ so that `isPrior` checks will be skipped, and
                // `hasMessageAvailableAsync` won't use `startMessageId_` in compare.
                startMessageId_ = std::nullopt;
            }
        } else {
            LOG_ERROR(getName() << "SeekStatus is not NOT_STARTED but lastSeekArg_ is not set");
        }
        return;
    } else if (subscriptionMode_ == Commands::SubscriptionModeDurable) {
        return;
    }

    Message nextMessageInQueue;
    if (incomingMessages_.peekAndClear(nextMessageInQueue)) {
        // There was at least one message pending in the queue
        const MessageId& nextMessageId = nextMessageInQueue.getMessageId();
        auto previousMessageId = (nextMessageId.batchIndex() >= 0)
                                     ? MessageIdBuilder()
                                           .ledgerId(nextMessageId.ledgerId())
                                           .entryId(nextMessageId.entryId())
                                           .batchIndex(nextMessageId.batchIndex() - 1)
                                           .batchSize(nextMessageId.batchSize())
                                           .build()
                                     : MessageIdBuilder()
                                           .ledgerId(nextMessageId.ledgerId())
                                           .entryId(nextMessageId.entryId() - 1)
                                           .build();
        startMessageId_ = previousMessageId;
    } else if (lastDequedMessageId_ != MessageId::earliest()) {
        // If the queue was empty we need to restart from the message just after the last one that has been
        // dequeued
        // in the past
        startMessageId_ = lastDequedMessageId_;
    }
}

void ConsumerImpl::increaseAvailablePermits(const ClientConnectionPtr& currentCnx, int delta) {
    int newAvailablePermits = availablePermits_.fetch_add(delta) + delta;

    while (newAvailablePermits >= receiverQueueRefillThreshold_ && messageListenerRunning_) {
        if (availablePermits_.compare_exchange_weak(newAvailablePermits, 0)) {
            sendFlowPermitsToBroker(currentCnx, newAvailablePermits);
            break;
        }
    }
}

void ConsumerImpl::increaseAvailablePermits(const Message& msg) {
    ClientConnectionPtr currentCnx = getCnx().lock();
    if (currentCnx && msg.impl_->cnx_ != currentCnx.get()) {
        LOG_DEBUG(getName() << "Not adding permit since connection is different.");
        return;
    }

    increaseAvailablePermits(currentCnx);
}

inline CommandSubscribe_SubType ConsumerImpl::getSubType() {
    ConsumerType type = config_.getConsumerType();
    switch (type) {
        case ConsumerExclusive:
            return CommandSubscribe_SubType_Exclusive;

        case ConsumerShared:
            return CommandSubscribe_SubType_Shared;

        case ConsumerFailover:
            return CommandSubscribe_SubType_Failover;

        case ConsumerKeyShared:
            return CommandSubscribe_SubType_Key_Shared;
    }
    BOOST_THROW_EXCEPTION(std::logic_error("Invalid ConsumerType enumeration value"));
}

inline CommandSubscribe_InitialPosition ConsumerImpl::getInitialPosition() {
    InitialPosition initialPosition = config_.getSubscriptionInitialPosition();
    switch (initialPosition) {
        case InitialPositionLatest:
            return CommandSubscribe_InitialPosition_Latest;

        case InitialPositionEarliest:
            return CommandSubscribe_InitialPosition_Earliest;
    }
    BOOST_THROW_EXCEPTION(std::logic_error("Invalid InitialPosition enumeration value"));
}

void ConsumerImpl::acknowledgeAsync(const MessageId& msgId, const ResultCallback& callback) {
    auto pair = prepareIndividualAck(msgId);
    const auto& msgIdToAck = pair.first;
    const bool readyToAck = pair.second;
    if (readyToAck) {
        ackGroupingTrackerPtr_->addAcknowledge(msgIdToAck, callback);
    } else {
        if (callback) {
            callback(ResultOk);
        }
    }
    interceptors_->onAcknowledge(Consumer(shared_from_this()), ResultOk, msgId);
}

void ConsumerImpl::acknowledgeAsync(const MessageIdList& messageIdList, const ResultCallback& callback) {
    MessageIdList messageIdListToAck;
    // TODO: Need to check if the consumer is ready. Same to all other public methods
    for (auto&& msgId : messageIdList) {
        auto pair = prepareIndividualAck(msgId);
        const auto& msgIdToAck = pair.first;
        const bool readyToAck = pair.second;
        if (readyToAck) {
            messageIdListToAck.emplace_back(msgIdToAck);
        }
        // Invoking `onAcknowledge` for all message ids no matter if it's ready to ack. This is consistent
        // with the Java client.
        interceptors_->onAcknowledge(Consumer(shared_from_this()), ResultOk, msgId);
    }
    this->ackGroupingTrackerPtr_->addAcknowledgeList(messageIdListToAck, callback);
}

void ConsumerImpl::acknowledgeCumulativeAsync(const MessageId& msgId, const ResultCallback& callback) {
    if (!isCumulativeAcknowledgementAllowed(config_.getConsumerType())) {
        interceptors_->onAcknowledgeCumulative(Consumer(shared_from_this()),
                                               ResultCumulativeAcknowledgementNotAllowedError, msgId);
        if (callback) {
            callback(ResultCumulativeAcknowledgementNotAllowedError);
        }
        return;
    }
    auto pair = prepareCumulativeAck(msgId);
    const auto& msgIdToAck = pair.first;
    const auto& readyToAck = pair.second;
    if (readyToAck) {
        consumerStatsBasePtr_->messageAcknowledged(ResultOk, CommandAck_AckType_Cumulative, 1);
        unAckedMessageTrackerPtr_->removeMessagesTill(msgIdToAck);
        ackGroupingTrackerPtr_->addAcknowledgeCumulative(msgIdToAck, callback);
    } else if (callback) {
        callback(ResultOk);
    }
    interceptors_->onAcknowledgeCumulative(Consumer(shared_from_this()), ResultOk, msgId);
}

bool ConsumerImpl::isCumulativeAcknowledgementAllowed(ConsumerType consumerType) {
    return consumerType != ConsumerKeyShared && consumerType != ConsumerShared;
}

std::pair<MessageId, bool> ConsumerImpl::prepareIndividualAck(const MessageId& messageId) {
    auto messageIdImpl = Commands::getMessageIdImpl(messageId);
    auto batchedMessageIdImpl = std::dynamic_pointer_cast<BatchedMessageIdImpl>(messageIdImpl);

    auto batchSize = messageId.batchSize();
    if (!batchedMessageIdImpl || batchedMessageIdImpl->ackIndividual(messageId.batchIndex())) {
        consumerStatsBasePtr_->messageAcknowledged(ResultOk, CommandAck_AckType_Individual,
                                                   (batchSize > 0) ? batchSize : 1);
        unAckedMessageTrackerPtr_->remove(messageId);
        possibleSendToDeadLetterTopicMessages_.remove(messageId);
        if (std::dynamic_pointer_cast<ChunkMessageIdImpl>(messageIdImpl)) {
            return std::make_pair(messageId, true);
        }
        return std::make_pair(discardBatch(messageId), true);
    } else if (config_.isBatchIndexAckEnabled()) {
        return std::make_pair(messageId, true);
    } else {
        return std::make_pair(MessageId{}, false);
    }
}

std::pair<MessageId, bool> ConsumerImpl::prepareCumulativeAck(const MessageId& messageId) {
    auto messageIdImpl = Commands::getMessageIdImpl(messageId);
    auto batchedMessageIdImpl = std::dynamic_pointer_cast<BatchedMessageIdImpl>(messageIdImpl);

    if (!batchedMessageIdImpl || batchedMessageIdImpl->ackCumulative(messageId.batchIndex())) {
        return std::make_pair(discardBatch(messageId), true);
    } else if (config_.isBatchIndexAckEnabled()) {
        return std::make_pair(messageId, true);
    } else {
        if (batchedMessageIdImpl->shouldAckPreviousMessageId()) {
            return std::make_pair(batchedMessageIdImpl->getPreviousMessageId(), true);
        } else {
            return std::make_pair(MessageId{}, false);
        }
    }
}

void ConsumerImpl::negativeAcknowledge(const MessageId& messageId) {
    unAckedMessageTrackerPtr_->remove(messageId);
    negativeAcksTracker_->add(messageId);
}

void ConsumerImpl::disconnectConsumer() { disconnectConsumer(std::nullopt); }

void ConsumerImpl::disconnectConsumer(const optional<std::string>& assignedBrokerUrl) {
    LOG_INFO("Broker notification of Closed consumer: "
             << consumerId_ << (assignedBrokerUrl ? (" assignedBrokerUrl: " + *assignedBrokerUrl) : ""));
    resetCnx();
    scheduleReconnection(assignedBrokerUrl);
}

void ConsumerImpl::closeAsync(const ResultCallback& originalCallback) {
    auto callback = [this, originalCallback](Result result, bool alreadyClosed = false) {
        internalShutdown();
        if (result == ResultOk) {
            if (!alreadyClosed) {
                LOG_INFO(getName() << "Closed consumer " << consumerId_);
            }
        } else {
            LOG_WARN(getName() << "Failed to close consumer: " << result);
        }
        if (originalCallback) {
            originalCallback(result);
        }
    };

    auto state = state_.load();
    if (state == Closing || state == Closed) {
        callback(ResultOk, true);
        return;
    }

    LOG_INFO(getName() << "Closing consumer for topic " << topic());
    state_ = Closing;
    incomingMessages_.close();

    // Flush pending grouped ACK requests.
    ackGroupingTrackerPtr_->close();
    negativeAcksTracker_->close();

    ClientConnectionPtr cnx = getCnx().lock();
    if (!cnx) {
        // If connection is gone, also the consumer is closed on the broker side
        callback(ResultOk);
        return;
    }

    ClientImplPtr client = client_.lock();
    if (!client) {
        // Client was already destroyed
        callback(ResultOk);
        return;
    }

    cancelTimers();

    auto requestId = newRequestId();
    auto self = get_shared_this_ptr();
    cnx->sendRequestWithId(Commands::newCloseConsumer(consumerId_, requestId), requestId, "CLOSE_CONSUMER")
        .addListener([self, callback](const Error& error, const ResponseData&) { callback(error.result); });
}

const std::string& ConsumerImpl::getName() const { return consumerStr_; }

void ConsumerImpl::shutdown() { internalShutdown(); }

void ConsumerImpl::internalShutdown() {
    ackGroupingTrackerPtr_->close();
    incomingMessages_.clear();
    possibleSendToDeadLetterTopicMessages_.clear();
    resetCnx();
    interceptors_->close();
    auto client = client_.lock();
    if (client) {
        client->cleanupConsumer(this);
    }
    negativeAcksTracker_->close();
    cancelTimers();
    consumerCreatedPromise_.setFailed(ResultAlreadyClosed);
    failPendingReceiveCallback();
    failPendingBatchReceiveCallback();
    state_ = Closed;
}

bool ConsumerImpl::isClosed() { return state_ == Closed; }

bool ConsumerImpl::isOpen() { return state_ == Ready; }

Result ConsumerImpl::pauseMessageListener() {
    if (!messageListener_) {
        return ResultInvalidConfiguration;
    }
    messageListenerRunning_ = false;
    return ResultOk;
}

Result ConsumerImpl::resumeMessageListener() {
    if (!messageListener_) {
        return ResultInvalidConfiguration;
    }

    if (messageListenerRunning_) {
        // Not paused
        return ResultOk;
    }
    messageListenerRunning_ = true;
    const size_t count = incomingMessages_.size();

    for (size_t i = 0; i < count; i++) {
        // Trigger message listener callback in a separate thread
        listenerExecutor_->postWork(std::bind(&ConsumerImpl::internalListener, get_shared_this_ptr()));
    }
    // Check current permits and determine whether to send FLOW command
    this->increaseAvailablePermits(getCnx().lock(), 0);
    return ResultOk;
}

void ConsumerImpl::redeliverUnacknowledgedMessages() {
    static std::set<MessageId> emptySet;
    redeliverMessages(emptySet);
    unAckedMessageTrackerPtr_->clear();
}

void ConsumerImpl::redeliverUnacknowledgedMessages(const std::set<MessageId>& messageIds) {
    if (messageIds.empty()) {
        return;
    }
    if (config_.getConsumerType() != ConsumerShared && config_.getConsumerType() != ConsumerKeyShared) {
        redeliverUnacknowledgedMessages();
        return;
    }

    ClientConnectionPtr cnx = getCnx().lock();
    if (cnx) {
        if (cnx->getServerProtocolVersion() >= proto::v2) {
            auto needRedeliverMsgs = std::make_shared<std::set<MessageId>>();
            auto needCallBack = std::make_shared<std::atomic<int>>(messageIds.size());
            auto self = get_shared_this_ptr();
            // TODO Support MAX_REDELIVER_UNACKNOWLEDGED Avoid redelivering too many messages
            for (const auto& msgId : messageIds) {
                processPossibleToDLQ(msgId,
                                     [self, needRedeliverMsgs, &msgId, needCallBack](bool processSuccess) {
                                         if (!processSuccess) {
                                             needRedeliverMsgs->emplace(msgId);
                                         }
                                         if (--(*needCallBack) == 0 && !needRedeliverMsgs->empty()) {
                                             self->redeliverMessages(*needRedeliverMsgs);
                                         }
                                     });
            }
        }
    } else {
        LOG_WARN("Connection not ready for Consumer - " << getConsumerId());
    }
}

void ConsumerImpl::redeliverMessages(const std::set<MessageId>& messageIds) {
    ClientConnectionPtr cnx = getCnx().lock();
    if (cnx) {
        if (cnx->getServerProtocolVersion() >= proto::v2) {
            cnx->sendCommand(Commands::newRedeliverUnacknowledgedMessages(consumerId_, messageIds));
            LOG_DEBUG("Sending RedeliverUnacknowledgedMessages command for Consumer - " << getConsumerId());
        }
    } else {
        LOG_DEBUG("Connection not ready for Consumer - " << getConsumerId());
    }
}

int ConsumerImpl::getNumOfPrefetchedMessages() const { return incomingMessages_.size(); }

void ConsumerImpl::getBrokerConsumerStatsAsync(const BrokerConsumerStatsCallback& callback) {
    if (state_ != Ready) {
        LOG_ERROR(getName() << "Client connection is not open, please try again later.")
        callback(ResultConsumerNotInitialized, BrokerConsumerStats());
        return;
    }

    Lock lock(mutex_);
    if (brokerConsumerStats_.isValid()) {
        LOG_DEBUG(getName() << "Serving data from cache");
        BrokerConsumerStatsImpl brokerConsumerStats = brokerConsumerStats_;
        lock.unlock();
        callback(ResultOk,
                 BrokerConsumerStats(std::make_shared<BrokerConsumerStatsImpl>(brokerConsumerStats_)));
        return;
    }
    lock.unlock();

    ClientConnectionPtr cnx = getCnx().lock();
    if (cnx) {
        if (cnx->getServerProtocolVersion() >= proto::v8) {
            auto requestId = newRequestId();
            LOG_DEBUG(getName() << " Sending ConsumerStats Command for Consumer - " << getConsumerId()
                                << ", requestId - " << requestId);

            cnx->newConsumerStats(consumerId_, requestId)
                .addListener(std::bind(&ConsumerImpl::brokerConsumerStatsListener, get_shared_this_ptr(),
                                       std::placeholders::_1, std::placeholders::_2, callback));
            return;
        } else {
            LOG_ERROR(getName() << " Operation not supported since server protobuf version "
                                << cnx->getServerProtocolVersion() << " is older than proto::v7");
            callback(ResultUnsupportedVersionError, BrokerConsumerStats());
            return;
        }
    }
    LOG_ERROR(getName() << " Client Connection not ready for Consumer");
    callback(ResultNotConnected, BrokerConsumerStats());
}

void ConsumerImpl::brokerConsumerStatsListener(Result res, BrokerConsumerStatsImpl brokerConsumerStats,
                                               const BrokerConsumerStatsCallback& callback) {
    if (res == ResultOk) {
        Lock lock(mutex_);
        brokerConsumerStats.setCacheTime(config_.getBrokerConsumerStatsCacheTimeInMs());
        brokerConsumerStats_ = brokerConsumerStats;
    }

    if (callback) {
        callback(res, BrokerConsumerStats(std::make_shared<BrokerConsumerStatsImpl>(brokerConsumerStats)));
    }
}

void ConsumerImpl::seekAsync(const MessageId& msgId, const ResultCallback& callback) {
    const auto state = state_.load();
    if (state == Closed || state == Closing) {
        LOG_ERROR(getName() << "Client connection already closed.");
        if (callback) {
            callback(ResultAlreadyClosed);
        }
        return;
    }

    const auto requestId = newRequestId();
    auto nonNullCallback = (callback != nullptr) ? callback : [](Result) {};
    seekAsyncInternal(requestId, Commands::newSeek(consumerId_, requestId, msgId), SeekArg{msgId},
                      std::move(nonNullCallback));
}

void ConsumerImpl::seekAsync(SeekTimestampType timestamp, const ResultCallback& callback) {
    const auto state = state_.load();
    if (state == Closed || state == Closing) {
        LOG_ERROR(getName() << "Client connection already closed.");
        if (callback) {
            callback(ResultAlreadyClosed);
        }
        return;
    }

    const auto requestId = newRequestId();
    auto nonNullCallback = (callback != nullptr) ? callback : [](Result) {};
    seekAsyncInternal(requestId, Commands::newSeek(consumerId_, requestId, timestamp), SeekArg{timestamp},
                      std::move(nonNullCallback));
}

bool ConsumerImpl::isReadCompacted() { return readCompacted_; }

void ConsumerImpl::hasMessageAvailableAsync(const HasMessageAvailableCallback& callback) {
    if (!incomingMessages_.empty()) {
        callback(ResultOk, true);
        return;
    }
    bool compareMarkDeletePosition;
    {
        LockGuard lock{mutex_};
        compareMarkDeletePosition =
            // there is no message received by consumer, so we cannot compare the last position with the last
            // received position
            lastDequedMessageId_ == MessageId::earliest() &&
            // If the start message id is latest, we should seek to the actual last message first.
            (startMessageId_.value_or(MessageId::earliest()) == MessageId::latest() ||
             // If there is a previous seek operation by timestamp, the start message id will be incorrect, so
             // we cannot compare the start position with the last position.
             (lastSeekArg_.has_value() && std::holds_alternative<SeekTimestampType>(lastSeekArg_.value())));
    }
    if (compareMarkDeletePosition) {
        auto self = get_shared_this_ptr();
        getLastMessageIdAsync([self, callback](Result result, const GetLastMessageIdResponse& response) {
            if (result != ResultOk) {
                callback(result, {});
                return;
            }
            bool lastSeekIsByTimestamp = false;
            {
                LockGuard lock{self->mutex_};
                if (self->lastSeekArg_.has_value() &&
                    std::holds_alternative<SeekTimestampType>(self->lastSeekArg_.value())) {
                    lastSeekIsByTimestamp = true;
                }
            }
            auto handleResponse = [self, lastSeekIsByTimestamp, response, callback] {
                if (response.hasMarkDeletePosition() && response.getLastMessageId().entryId() >= 0) {
                    // We only care about comparing ledger ids and entry ids as mark delete position
                    // doesn't have other ids such as batch index
                    auto compareResult = compareLedgerAndEntryId(response.getMarkDeletePosition(),
                                                                 response.getLastMessageId());
                    // When the consumer has sought by timestamp, broker will ignore the
                    // startMessageIdInclusive config, so the compare should still be exclusive
                    if (lastSeekIsByTimestamp || !self->config_.isStartMessageIdInclusive()) {
                        callback(ResultOk, compareResult < 0);
                    } else {
                        callback(ResultOk, compareResult <= 0);
                    }
                } else {
                    callback(ResultOk, false);
                }
            };
            if (self->config_.isStartMessageIdInclusive() && !lastSeekIsByTimestamp) {
                self->seekAsync(response.getLastMessageId(), [callback, handleResponse](Result result) {
                    if (result != ResultOk) {
                        callback(result, {});
                        return;
                    }
                    handleResponse();
                });
            } else {
                handleResponse();
            }
        });
    } else {
        if (hasMoreMessages()) {
            callback(ResultOk, true);
            return;
        }
        auto self = get_shared_this_ptr();
        getLastMessageIdAsync(
            [this, self, callback](Result result, const GetLastMessageIdResponse& response) {
                callback(result, (result == ResultOk) && hasMoreMessages());
            });
    }
}

void ConsumerImpl::getLastMessageIdAsync(const BrokerGetLastMessageIdCallback& callback) {
    const auto state = state_.load();
    if (state == Closed || state == Closing) {
        LOG_ERROR(getName() << "Client connection already closed.");
        if (callback) {
            callback(ResultAlreadyClosed, GetLastMessageIdResponse());
        }
        return;
    }

    TimeDuration operationTimeout = seconds(client_.lock()->conf().getOperationTimeoutSeconds());
    BackoffPtr backoff = std::make_shared<Backoff>(milliseconds(100), operationTimeout * 2, milliseconds(0));
    DeadlineTimerPtr timer = executor_->createDeadlineTimer();

    internalGetLastMessageIdAsync(backoff, operationTimeout, timer, callback);
}

void ConsumerImpl::internalGetLastMessageIdAsync(const BackoffPtr& backoff, TimeDuration remainTime,
                                                 const DeadlineTimerPtr& timer,
                                                 const BrokerGetLastMessageIdCallback& callback) {
    ClientConnectionPtr cnx = getCnx().lock();
    if (cnx) {
        if (cnx->getServerProtocolVersion() >= proto::v12) {
            auto requestId = newRequestId();
            LOG_DEBUG(getName() << " Sending getLastMessageId Command for Consumer - " << getConsumerId()
                                << ", requestId - " << requestId);

            auto self = get_shared_this_ptr();
            cnx->newGetLastMessageId(consumerId_, requestId)
                .addListener([this, self, callback](Result result, const GetLastMessageIdResponse& response) {
                    if (result == ResultOk) {
                        LOG_DEBUG(getName() << "getLastMessageId: " << response);
                        {
                            LockGuard lock{mutex_};
                            lastMessageIdInBroker_ = response.getLastMessageId();
                        }
                    } else {
                        LOG_ERROR(getName() << "Failed to getLastMessageId: " << result);
                    }
                    callback(result, response);
                });
        } else {
            LOG_ERROR(getName() << " Operation not supported since server protobuf version "
                                << cnx->getServerProtocolVersion() << " is older than proto::v12");
            callback(ResultUnsupportedVersionError, MessageId());
        }
    } else {
        TimeDuration next = std::min(remainTime, backoff->next());
        if (toMillis(next) <= 0) {
            LOG_ERROR(getName() << " Client Connection not ready for Consumer");
            callback(ResultNotConnected, MessageId());
            return;
        }
        remainTime -= next;

        timer->expires_after(next);

        auto self = shared_from_this();
        timer->async_wait([this, backoff, remainTime, timer, next, callback,
                           self](const ASIO_ERROR& ec) -> void {
            if (ec == ASIO::error::operation_aborted) {
                LOG_DEBUG(getName() << " Get last message id operation was cancelled, code[" << ec << "].");
                return;
            }
            if (ec) {
                LOG_ERROR(getName() << " Failed to get last message id, code[" << ec << "].");
                return;
            }
            LOG_WARN(getName() << " Could not get connection while getLastMessageId -- Will try again in "
                               << toMillis(next) << " ms")
            this->internalGetLastMessageIdAsync(backoff, remainTime, timer, callback);
        });
    }
}

void ConsumerImpl::setNegativeAcknowledgeEnabledForTesting(bool enabled) {
    negativeAcksTracker_->setEnabledForTesting(enabled);
}

void ConsumerImpl::trackMessage(const MessageId& messageId) {
    if (hasParent_) {
        unAckedMessageTrackerPtr_->remove(messageId);
    } else {
        unAckedMessageTrackerPtr_->add(messageId);
    }
}

bool ConsumerImpl::isConnected() const { return !getCnx().expired() && state_ == Ready; }

uint64_t ConsumerImpl::getNumberOfConnectedConsumer() { return isConnected() ? 1 : 0; }

void ConsumerImpl::seekAsyncInternal(long requestId, const SharedBuffer& seek, const SeekArg& seekArg,
                                     ResultCallback&& callback) {
    ClientConnectionPtr cnx = getCnx().lock();
    if (!cnx) {
        LOG_ERROR(getName() << " Client Connection not ready for Consumer");
        callback(ResultNotConnected);
        return;
    }
    bool hasPendingSeek = false;
    // Save the previous last seek arg in case seek failed
    decltype(lastSeekArg_) previousLastSeekArg;
    {
        std::lock_guard<std::mutex> lock(mutex_);
        if (seekStatus_ != SeekStatus::NOT_STARTED) {
            hasPendingSeek = true;
        } else {
            seekStatus_ = SeekStatus::IN_PROGRESS;
            if (seekCallback_.has_value()) {
                // This should never happen
                LOG_ERROR(getName() << "Previous seek callback is not triggered unexpectedly");
                executor_->postWork([callback{std::exchange(seekCallback_, std::nullopt).value()}] {
                    callback(ResultTimeout);
                });
            }
            seekCallback_ = std::move(callback);
            previousLastSeekArg = lastSeekArg_;
            lastSeekArg_ = seekArg;
        }
    }
    if (hasPendingSeek) {
        std::visit(
            [this](auto&& arg) {
                LOG_ERROR(getName() << "Attempted to seek " << arg << " when there is a pending seek");
            },
            seekArg);
        callback(ResultNotAllowedError);
        return;
    }

    std::visit([this](auto&& arg) { LOG_INFO(getName() << "Seeking subscription to " << arg); }, seekArg);

    auto weakSelf = weak_from_this();

    cnx->sendRequestWithId(seek, requestId, "SEEK")
        .addListener([this, weakSelf, previousLastSeekArg](const Error& error,
                                                           const ResponseData& responseData) {
            auto self = weakSelf.lock();
            if (!self) {
                return;
            }
            if (error.result == ResultOk) {
                LockGuard lock(mutex_);
                if (getCnx().expired() || reconnectionPending_) {
                    // Reconnection path: delay the seek callback until connectionOpened. clearReceiveQueue()
                    // and handleCreateConsumer() (which clears incomingMessages_ under the lock) run before
                    // the seek callback is invoked, so hasMessageAvailable() after seek sees cleared state.
                    seekStatus_ = SeekStatus::COMPLETED;
                    LOG_INFO(getName() << "Delay the seek future until the reconnection is done");
                } else {
                    LOG_INFO(getName() << "Seek successfully");
                    ackGroupingTrackerPtr_->flushAndClean();
                    incomingMessages_.clear();
                    if (lastSeekArg_.has_value() && std::holds_alternative<MessageId>(lastSeekArg_.value())) {
                        startMessageId_ = std::get<MessageId>(lastSeekArg_.value());
                    }
                    if (!seekCallback_.has_value()) {
                        LOG_ERROR(getName() << "Seek callback is not set");
                        return;
                    }
                    executor_->postWork(
                        [self, callback{std::exchange(seekCallback_, std::nullopt).value()}]() {
                            callback(ResultOk);
                        });
                    seekStatus_ = SeekStatus::NOT_STARTED;
                }  // else: complete the seek future after connection is established
            } else {
                LOG_ERROR(getName() << "Failed to seek: " << error.result);
                LockGuard lock{mutex_};
                seekStatus_ = SeekStatus::NOT_STARTED;
                lastSeekArg_ = previousLastSeekArg;
                executor_->postWork([self, callback{std::exchange(seekCallback_, std::nullopt).value()},
                                     result{error.result}]() { callback(result); });
            }
        });
}

bool ConsumerImpl::hasEnoughMessagesForBatchReceive() const {
    if (batchReceivePolicy_.getMaxNumMessages() <= 0 && batchReceivePolicy_.getMaxNumBytes() <= 0) {
        return false;
    }
    return (batchReceivePolicy_.getMaxNumMessages() > 0 &&
            incomingMessages_.size() >= batchReceivePolicy_.getMaxNumMessages()) ||
           (batchReceivePolicy_.getMaxNumBytes() > 0 &&
            incomingMessagesSize_ >= batchReceivePolicy_.getMaxNumBytes());
}

std::shared_ptr<ConsumerImpl> ConsumerImpl::get_shared_this_ptr() {
    return std::dynamic_pointer_cast<ConsumerImpl>(shared_from_this());
}

void ConsumerImpl::cancelTimers() noexcept {
    cancelTimer(*batchReceiveTimer_);
    cancelTimer(*checkExpiredChunkedTimer_);
    unAckedMessageTrackerPtr_->stop();
    consumerStatsBasePtr_->stop();
}

void ConsumerImpl::processPossibleToDLQ(const MessageId& messageId, const ProcessDLQCallBack& cb) {
    auto messages = possibleSendToDeadLetterTopicMessages_.find(messageId);
    if (!messages) {
        cb(false);
        return;
    }

    // Initialize deadLetterProducer_
    if (!deadLetterProducer_) {
        std::lock_guard<std::mutex> createLock(createProducerLock_);
        if (!deadLetterProducer_) {
            deadLetterProducer_ = std::make_shared<Promise<Result, Producer>>();
            ProducerConfiguration producerConfiguration;
            producerConfiguration.setSchema(config_.getSchema());
            producerConfiguration.setBlockIfQueueFull(false);
            producerConfiguration.setBatchingEnabled(false);
            producerConfiguration.impl_->initialSubscriptionName =
                deadLetterPolicy_.getInitialSubscriptionName();
            ClientImplPtr client = client_.lock();
            if (client) {
                auto self = get_shared_this_ptr();
                client->createProducerAsync(
                    deadLetterPolicy_.getDeadLetterTopic(), producerConfiguration, [self](const auto& v) {
                        if (const auto* producer = std::get_if<Producer>(&v)) {
                            self->deadLetterProducer_->setValue(*producer);
                        } else {
                            LOG_ERROR("Dead letter producer create exception with topic: "
                                      << self->deadLetterPolicy_.getDeadLetterTopic()
                                      << " ex: " << std::get<Error>(v).result);
                            self->deadLetterProducer_.reset();
                        }
                    });
            } else {
                LOG_WARN(getName() << "Client is destroyed and cannot create dead letter producer.");
                return;
            }
        }
    }

    for (const auto& message : messages.value()) {
        auto weakSelf = weak_from_this();
        deadLetterProducer_->getFuture().addListener([this, weakSelf, message, messageId, cb](
                                                         Result res, Producer producer) {
            auto self = weakSelf.lock();
            if (!self) {
                return;
            }
            auto originMessageId = message.getMessageId();
            std::stringstream originMessageIdStr;
            originMessageIdStr << originMessageId;
            MessageBuilder msgBuilder;
            msgBuilder.setAllocatedContent(const_cast<void*>(message.getData()), message.getLength())
                .setProperties(message.getProperties())
                .setProperty(PROPERTY_ORIGIN_MESSAGE_ID, originMessageIdStr.str())
                .setProperty(SYSTEM_PROPERTY_REAL_TOPIC, message.getTopicName());
            if (message.hasPartitionKey()) {
                msgBuilder.setPartitionKey(message.getPartitionKey());
            }
            if (message.hasOrderingKey()) {
                msgBuilder.setOrderingKey(message.getOrderingKey());
            }
            producer.sendAsync(msgBuilder.build(), [this, weakSelf, originMessageId, messageId, cb](
                                                       Result res, const MessageId& messageIdInDLQ) {
                auto self = weakSelf.lock();
                if (!self) {
                    return;
                }
                if (res == ResultOk) {
                    if (state_ != Ready) {
                        LOG_WARN(
                            "Send to the DLQ successfully, but consumer is not ready. ignore acknowledge : "
                            << state_);
                        cb(false);
                        return;
                    }
                    possibleSendToDeadLetterTopicMessages_.remove(messageId);
                    acknowledgeAsync(originMessageId, [this, weakSelf, originMessageId, cb](Result result) {
                        auto self = weakSelf.lock();
                        if (!self) {
                            return;
                        }
                        if (result != ResultOk) {
                            LOG_WARN("{" << topic() << "} {" << subscription_ << "} {" << getConsumerName()
                                         << "} Failed to acknowledge the message {" << originMessageId
                                         << "} of the original topic but send to the DLQ successfully : "
                                         << result);
                            cb(false);
                        } else {
                            LOG_DEBUG("Send msg:" << originMessageId
                                                  << "to DLQ success and acknowledge success.");
                            cb(true);
                        }
                    });
                } else {
                    LOG_WARN("{" << topic() << "} {" << subscription_ << "} {" << getConsumerName()
                                 << "} Failed to send DLQ message to {"
                                 << deadLetterPolicy_.getDeadLetterTopic() << "} for message id "
                                 << "{" << originMessageId << "} : " << res);
                    cb(false);
                }
            });
        });
    }
}

void ConsumerImpl::doImmediateAck(const ClientConnectionPtr& cnx, const MessageId& msgId,
                                  CommandAck_AckType ackType, const ResultCallback& callback) {
    const auto& ackSet = Commands::getMessageIdImpl(msgId)->getBitSet();
    if (config_.isAckReceiptEnabled()) {
        auto requestId = newRequestId();
        cnx->sendRequestWithId(
               Commands::newAck(consumerId_, msgId.ledgerId(), msgId.entryId(), ackSet, ackType, requestId),
               requestId, "ACK")
            .addListener([callback](const Error& error, const ResponseData&) {
                if (callback) {
                    callback(error.result);
                }
            });
    } else {
        cnx->sendCommand(Commands::newAck(consumerId_, msgId.ledgerId(), msgId.entryId(), ackSet, ackType));
        if (callback) {
            callback(ResultOk);
        }
    }
}

void ConsumerImpl::doImmediateAck(const ClientConnectionPtr& cnx, const std::set<MessageId>& msgIds,
                                  const ResultCallback& callback) {
    std::set<MessageId> ackMsgIds;

    for (const auto& msgId : msgIds) {
        if (auto chunkMessageId =
                std::dynamic_pointer_cast<ChunkMessageIdImpl>(Commands::getMessageIdImpl(msgId))) {
            auto msgIdList = chunkMessageId->getChunkedMessageIds();
            ackMsgIds.insert(msgIdList.begin(), msgIdList.end());
        } else {
            ackMsgIds.insert(msgId);
        }
    }
    if (Commands::peerSupportsMultiMessageAcknowledgement(cnx->getServerProtocolVersion())) {
        if (config_.isAckReceiptEnabled()) {
            auto requestId = newRequestId();
            cnx->sendRequestWithId(Commands::newMultiMessageAck(consumerId_, ackMsgIds, requestId), requestId,
                                   "ACK")
                .addListener([callback](const Error& error, const ResponseData&) {
                    if (callback) {
                        callback(error.result);
                    }
                });
        } else {
            cnx->sendCommand(Commands::newMultiMessageAck(consumerId_, ackMsgIds));
            if (callback) {
                callback(ResultOk);
            }
        }
    } else {
        auto count = std::make_shared<std::atomic<size_t>>(ackMsgIds.size());
        auto wrappedCallback = [callback, count](Result result) {
            if (--*count == 0 && callback) {
                callback(result);
            }
        };
        for (auto&& msgId : ackMsgIds) {
            doImmediateAck(msgId, wrappedCallback, CommandAck_AckType_Individual);
        }
    }
}

void ConsumerImpl::doImmediateAck(const MessageId& msgId, const ResultCallback& callback,
                                  CommandAck_AckType ackType) {
    const auto cnx = getCnx().lock();
    if (!cnx) {
        if (callback) {
            callback(ResultAlreadyClosed);
        }
        return;
    }
    if (ackType == CommandAck_AckType_Individual) {
        // If it's individual ack, we need to acknowledge all message IDs in a chunked message Id
        // If it's cumulative ack, we only need to ack the last message ID of a chunked message.
        // ChunkedMessageId return last chunk message ID by default, so we don't need to handle it.
        if (auto chunkMessageId =
                std::dynamic_pointer_cast<ChunkMessageIdImpl>(Commands::getMessageIdImpl(msgId))) {
            auto msgIdList = chunkMessageId->getChunkedMessageIds();
            doImmediateAck(cnx, std::set<MessageId>(msgIdList.begin(), msgIdList.end()), callback);
            return;
        }
    }
    doImmediateAck(cnx, msgId, ackType, callback);
}

void ConsumerImpl::doImmediateAck(const std::set<MessageId>& msgIds, const ResultCallback& callback) {
    const auto cnx = getCnx().lock();
    if (!cnx) {
        if (callback) {
            callback(ResultAlreadyClosed);
        }
        return;
    }
    doImmediateAck(cnx, msgIds, callback);
}

} /* namespace pulsar */
