/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.contract.s3a;

import java.io.FileNotFoundException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractMultipartUploaderTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.impl.ChecksumSupport;
import org.apache.hadoop.test.tags.IntegrationTest;
import org.apache.hadoop.test.tags.ScaleTest;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.Constants.CHECKSUM_ALGORITHM;
import static org.apache.hadoop.fs.s3a.Constants.CHECKSUM_GENERATION;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.DEFAULT_MULTIPART_COMMIT_CONSUMES_UPLOAD_ID;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.DEFAULT_SCALE_TESTS_ENABLED;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_HUGE_PARTITION_SIZE;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_SCALE_TESTS_ENABLED;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.MULTIPART_COMMIT_CONSUMES_UPLOAD_ID;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT_MILLIS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeMultipartUploads;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeNotS3ExpressFileSystem;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBool;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBytes;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.impl.ChecksumSupport.getChecksumAlgorithm;
import static org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles.DEFAULT_HUGE_PARTITION_SIZE;

/**
 * Test MultipartUploader with S3A.
 * <p></p>
 * Although not an S3A Scale test subclass, it uses the -Dscale option
 * to enable it, and partition size option to control the size of
 * parts uploaded.
 */
@IntegrationTest
@ScaleTest
public class ITestS3AContractMultipartUploader extends
    AbstractContractMultipartUploaderTest {

  private int partitionSize;

  private boolean mpuCommitConsumesUploadId;

  /**
   * S3 requires a minimum part size of 5MB (except the last part).
   * @return 5MB+ value
   */
  @Override
  protected int partSizeInBytes() {
    return partitionSize;
  }

  @Override
  protected int getTestPayloadCount() {
    return 3;
  }

  @Override
  public S3AFileSystem getFileSystem() {
    return (S3AFileSystem) super.getFileSystem();
  }

  @Override
  protected AbstractFSContract createContract(Configuration conf) {
    return new S3AContract(conf);
  }

  /**
   * Bigger test: use the scale timeout.
   * @return the timeout for scale tests.
   */
  @Override
  protected int getTestTimeoutMillis() {
    return SCALE_TEST_TIMEOUT_MILLIS;
  }


  @Override
  protected boolean supportsConcurrentUploadsToSamePath() {
    return true;
  }

  @Override
  protected boolean finalizeConsumesUploadIdImmediately() {
    return mpuCommitConsumesUploadId;
  }

  @Override
  protected Configuration createConfiguration() {
    final Configuration conf = super.createConfiguration();
    // use whatever the default checksum generation option is.
    removeBaseAndBucketOverrides(conf, CHECKSUM_GENERATION, CHECKSUM_ALGORITHM);
    conf.setBoolean(CHECKSUM_GENERATION, false);
    conf.set(CHECKSUM_ALGORITHM, ChecksumSupport.NONE);
    disableFilesystemCaching(conf);
    return conf;
  }

  @BeforeEach
  @Override
  public void setup() throws Exception {
    super.setup();
    Configuration conf = getContract().getConf();
    boolean enabled = getTestPropertyBool(
        conf,
        KEY_SCALE_TESTS_ENABLED,
        DEFAULT_SCALE_TESTS_ENABLED);
    assume("Scale test disabled: to enable set property " +
            KEY_SCALE_TESTS_ENABLED,
        enabled);
    final Configuration fsConf = getFileSystem().getConf();
    assumeMultipartUploads(fsConf);
    partitionSize = (int) getTestPropertyBytes(conf,
        KEY_HUGE_PARTITION_SIZE,
        DEFAULT_HUGE_PARTITION_SIZE);
    mpuCommitConsumesUploadId = fsConf.getBoolean(
        MULTIPART_COMMIT_CONSUMES_UPLOAD_ID,
        DEFAULT_MULTIPART_COMMIT_CONSUMES_UPLOAD_ID);
    LOG.info("{} = {}", MULTIPART_COMMIT_CONSUMES_UPLOAD_ID, mpuCommitConsumesUploadId);
    LOG.info("{} = {}", CHECKSUM_ALGORITHM, getChecksumAlgorithm(fsConf));
  }

  /**
   * S3 has no concept of directories, so this test does not apply.
   */
  @Test
  @Override
  public void testDirectoryInTheWay() throws Exception {
    skip("Unsupported");
  }

  @Test
  @Override
  public void testMultipartUploadReverseOrder() throws Exception {
    skip("skipped for speed");
  }

  @Test
  @Override
  public void testMultipartUploadReverseOrderNonContiguousPartNumbers() throws Exception {
    assumeNotS3ExpressFileSystem(getFileSystem());
    final Configuration fsConf = getFileSystem().getConf();
    super.testMultipartUploadReverseOrderNonContiguousPartNumbers();
  }

  @Test
  @Override
  public void testConcurrentUploads() throws Throwable {
    assumeNotS3ExpressFileSystem(getFileSystem());
    super.testConcurrentUploads();
  }

  @Test
  @Override
  public void testMultipartUploadAbort() throws Exception {
    try {
      super.testMultipartUploadAbort();
    } catch (FileNotFoundException e) {
      LOG.info("Multipart upload not found in abort()."
          + " This is common on third-party stores: {}",
          e.toString());
      LOG.debug("Exception: ", e);
    }
  }
}
