/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.http.client;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.CommonPathCapabilities;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.http.server.HttpFSAuthenticationFilter;
import org.apache.hadoop.fs.http.server.HttpFSServerWebApp;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.test.HFSTestCase;
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.test.TestDir;
import org.apache.hadoop.test.TestDirHelper;
import org.apache.hadoop.test.TestHdfs;
import org.apache.hadoop.test.TestHdfsHelper;
import org.apache.hadoop.test.TestJetty;
import org.apache.hadoop.test.TestJettyHelper;
import org.apache.hadoop.util.Lists;

import org.json.simple.JSONObject;
import org.json.simple.parser.ContainerFactory;
import org.json.simple.parser.JSONParser;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Writer;
import java.net.URI;
import java.net.URL;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.concurrent.atomic.AtomicReference;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeFalse;

public abstract class BaseTestHttpFSWith extends HFSTestCase {
  protected abstract Path getProxiedFSTestDir();

  protected abstract String getProxiedFSURI();

  protected abstract Configuration getProxiedFSConf();

  protected boolean isLocalFS() {
    return getProxiedFSURI().startsWith("file://");
  }

  private void createHttpFSServer() throws Exception {
    File homeDir = TestDirHelper.getTestDir();
    assertTrue(new File(homeDir, "conf").mkdir());
    assertTrue(new File(homeDir, "log").mkdir());
    assertTrue(new File(homeDir, "temp").mkdir());
    HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath());

    File secretFile = new File(new File(homeDir, "conf"), "secret");
    Writer w = new FileWriter(secretFile);
    w.write("secret");
    w.close();

    //FileSystem being served by HttpFS
    String fsDefaultName = getProxiedFSURI();
    Configuration conf = new Configuration(false);
    conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName);
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
    // For BaseTestHttpFSWith#testFileAclsCustomizedUserAndGroupNames
    conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
        "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$");
    conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
        "^(default:)?(user|group|mask|other):" +
            "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" +
            "(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:" +
            "([rwx-]{3})?)*$");
    File hdfsSite = new File(new File(homeDir, "conf"), "hdfs-site.xml");
    OutputStream os = new FileOutputStream(hdfsSite);
    conf.writeXml(os);
    os.close();

    //HTTPFS configuration
    conf = new Configuration(false);
    conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups",
             HadoopUsersConfTestHelper.getHadoopProxyUserGroups());
    conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts",
             HadoopUsersConfTestHelper.getHadoopProxyUserHosts());
    conf.set(HttpFSAuthenticationFilter.HADOOP_HTTP_CONF_PREFIX +
        AuthenticationFilter.SIGNATURE_SECRET_FILE, secretFile.getAbsolutePath());
    File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml");
    os = new FileOutputStream(httpfsSite);
    conf.writeXml(os);
    os.close();

    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    URL url = cl.getResource("webapp");
    WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs");
    Server server = TestJettyHelper.getJettyServer();
    server.setHandler(context);
    server.start();
  }

  protected Class getFileSystemClass() {
    return HttpFSFileSystem.class;
  }

  protected String getScheme() {
    return "webhdfs";
  }

  protected FileSystem getHttpFSFileSystem(Configuration conf) throws
      Exception {
    conf.set("fs.webhdfs.impl", getFileSystemClass().getName());
    URI uri = new URI(getScheme() + "://" +
                      TestJettyHelper.getJettyURL().toURI().getAuthority());
    return FileSystem.get(uri, conf);
  }

  protected FileSystem getHttpFSFileSystem() throws Exception {
    Configuration conf = new Configuration();
    return getHttpFSFileSystem(conf);
  }

  protected void testGet() throws Exception {
    FileSystem fs = getHttpFSFileSystem();
    assertNotNull(fs);
    URI uri = new URI(getScheme() + "://" +
                      TestJettyHelper.getJettyURL().toURI().getAuthority());
    assertEquals(fs.getUri(), uri);
    fs.close();
  }

  private void testOpen() throws Exception {
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    Path path = new Path(getProxiedFSTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);
    os.close();
    fs.close();
    fs = getHttpFSFileSystem();
    InputStream is = fs.open(new Path(path.toUri().getPath()));
    assertEquals(is.read(), 1);
    is.close();
    fs.close();
  }

  private void testCreate(Path path, boolean override) throws Exception {
    FileSystem fs = getHttpFSFileSystem();
    FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE);
    OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024,
                                (short) 2, 100 * 1024 * 1024, null);
    os.write(1);
    os.close();
    fs.close();

    fs = FileSystem.get(getProxiedFSConf());
    FileStatus status = fs.getFileStatus(path);
    if (!isLocalFS()) {
      assertEquals(status.getReplication(), 2);
      assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
    }
    assertEquals(status.getPermission(), permission);
    InputStream is = fs.open(path);
    assertEquals(is.read(), 1);
    is.close();
    fs.close();
  }

  private void testCreate() throws Exception {
    Path path = new Path(getProxiedFSTestDir(), "foo.txt");
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    fs.delete(path, true);
    testCreate(path, false);
    testCreate(path, true);
    try {
      testCreate(path, false);
      fail("the create should have failed because the file exists " +
          "and override is FALSE");
    } catch (IOException ex) {
      System.out.println("#");
    } catch (Exception ex) {
      fail(ex.toString());
    }
  }

  private void testAppend() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      fs.close();
      fs = getHttpFSFileSystem();
      os = fs.append(new Path(path.toUri().getPath()));
      os.write(2);
      os.close();
      fs.close();
      fs = FileSystem.get(getProxiedFSConf());
      InputStream is = fs.open(path);
      assertEquals(is.read(), 1);
      assertEquals(is.read(), 2);
      assertEquals(is.read(), -1);
      is.close();
      fs.close();
    }
  }

  private void testTruncate() throws Exception {
    if (!isLocalFS()) {
      final short repl = 3;
      final int blockSize = 1024;
      final int numOfBlocks = 2;
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path file = new Path(getProxiedFSTestDir(), "foo.txt");
      final byte[] data = FileSystemTestHelper.getFileData(
          numOfBlocks, blockSize);
      FileSystemTestHelper.createFile(fs, file, data, blockSize, repl);

      final int newLength = blockSize;

      boolean isReady = fs.truncate(file, newLength);
      assertTrue(isReady, "Recovery is not expected.");

      FileStatus fileStatus = fs.getFileStatus(file);
      assertEquals(fileStatus.getLen(), newLength);
      AppendTestUtil.checkFullFile(fs, file, newLength, data, file.toString());

      fs.close();
      assertPathCapabilityForTruncate(file);
    }
  }

  private void assertPathCapabilityForTruncate(Path file) throws Exception {
    FileSystem fs = this.getHttpFSFileSystem();
    assertTrue(fs.hasPathCapability(file, CommonPathCapabilities.FS_TRUNCATE),
        "HttpFS/WebHdfs/SWebHdfs support truncate");
    fs.close();
  }

  private void testConcat() throws Exception {
    Configuration config = getProxiedFSConf();
    config.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(config);
      fs.mkdirs(getProxiedFSTestDir());
      Path path1 = new Path("/test/foo.txt");
      Path path2 = new Path("/test/bar.txt");
      Path path3 = new Path("/test/derp.txt");
      DFSTestUtil.createFile(fs, path1, 1024, (short) 3, 0);
      DFSTestUtil.createFile(fs, path2, 1024, (short) 3, 0);
      DFSTestUtil.createFile(fs, path3, 1024, (short) 3, 0);
      fs.close();
      fs = getHttpFSFileSystem();
      fs.concat(path1, new Path[]{path2, path3});
      fs.close();
      fs = FileSystem.get(config);
      assertTrue(fs.exists(path1));
      assertFalse(fs.exists(path2));
      assertFalse(fs.exists(path3));
      fs.close();
    }
  }

  private void testRename() throws Exception {
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    Path path = new Path(getProxiedFSTestDir(), "foo");
    fs.mkdirs(path);
    fs.close();
    fs = getHttpFSFileSystem();
    Path oldPath = new Path(path.toUri().getPath());
    Path newPath = new Path(path.getParent(), "bar");
    fs.rename(oldPath, newPath);
    fs.close();
    fs = FileSystem.get(getProxiedFSConf());
    assertFalse(fs.exists(oldPath));
    assertTrue(fs.exists(newPath));
    fs.close();
  }

  private void testDelete() throws Exception {
    Path foo = new Path(getProxiedFSTestDir(), "foo");
    Path bar = new Path(getProxiedFSTestDir(), "bar");
    Path foe = new Path(getProxiedFSTestDir(), "foe");
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    fs.mkdirs(foo);
    fs.mkdirs(new Path(bar, "a"));
    fs.mkdirs(foe);

    FileSystem hoopFs = getHttpFSFileSystem();
    assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
    assertFalse(fs.exists(foo));
    try {
      hoopFs.delete(new Path(bar.toUri().getPath()), false);
      fail();
    } catch (IOException ex) {
    } catch (Exception ex) {
      fail();
    }
    assertTrue(fs.exists(bar));
    assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
    assertFalse(fs.exists(bar));

    assertTrue(fs.exists(foe));
    assertTrue(hoopFs.delete(foe, true));
    assertFalse(fs.exists(foe));

    hoopFs.close();
    fs.close();
  }

  private void testListSymLinkStatus() throws Exception {
    if (isLocalFS()) {
      // do not test the the symlink for local FS.
      return;
    }
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    boolean isWebhdfs = fs instanceof WebHdfsFileSystem;
    Path path =
        new Path(getProxiedFSTestDir() + "-symlink", "targetFoo.txt");
    OutputStream os = fs.create(path);
    os.write(1);
    os.close();
    Path linkPath =
        new Path(getProxiedFSTestDir()+ "-symlink", "symlinkFoo.txt");
    fs.createSymlink(path, linkPath, false);
    fs = getHttpFSFileSystem();
    FileStatus linkStatus = fs.getFileStatus(linkPath);
    FileStatus status1 = fs.getFileStatus(path);

    FileStatus[] stati = fs.listStatus(path.getParent());
    assertEquals(2, stati.length);

    int countSymlink = 0;
    for (int i = 0; i < stati.length; i++) {
      FileStatus fStatus = stati[i];
      countSymlink += fStatus.isSymlink() ? 1 : 0;
    }
    assertEquals(1, countSymlink);

    assertFalse(status1.isSymlink());
    if (isWebhdfs) {
      assertTrue(linkStatus.isSymlink());
    }
    fs.close();
  }

  private void testListStatus() throws Exception {
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    boolean isDFS = fs instanceof DistributedFileSystem;
    Path path = new Path(getProxiedFSTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);
    os.close();
    FileStatus status1 = fs.getFileStatus(path);
    fs.close();

    fs = getHttpFSFileSystem();
    FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath()));
    fs.close();

    assertEquals(status2.getPermission(), status1.getPermission());
    assertEquals(status2.getPath().toUri().getPath(),
        status1.getPath().toUri().getPath());
    assertEquals(status2.getReplication(), status1.getReplication());
    assertEquals(status2.getBlockSize(), status1.getBlockSize());
    assertEquals(status2.getAccessTime(), status1.getAccessTime());
    assertEquals(status2.getModificationTime(), status1.getModificationTime());
    assertEquals(status2.getOwner(), status1.getOwner());
    assertEquals(status2.getGroup(), status1.getGroup());
    assertEquals(status2.getLen(), status1.getLen());
    if (isDFS && status2 instanceof HdfsFileStatus) {
      assertTrue(status1 instanceof HdfsFileStatus);
      HdfsFileStatus hdfsFileStatus1 = (HdfsFileStatus) status1;
      HdfsFileStatus hdfsFileStatus2 = (HdfsFileStatus) status2;
      // Check HDFS-specific fields
      assertEquals(hdfsFileStatus2.getChildrenNum(),
          hdfsFileStatus1.getChildrenNum());
      assertEquals(hdfsFileStatus2.getFileId(),
          hdfsFileStatus1.getFileId());
      assertEquals(hdfsFileStatus2.getStoragePolicy(),
          hdfsFileStatus1.getStoragePolicy());
    }

    FileStatus[] stati = fs.listStatus(path.getParent());
    assertEquals(1, stati.length);
    assertEquals(stati[0].getPath().getName(), path.getName());

    // The full path should be the path to the file. See HDFS-12139
    FileStatus[] statl = fs.listStatus(path);
    assertEquals(1, statl.length);
    assertEquals(status2.getPath(), statl[0].getPath());
    assertEquals(statl[0].getPath().getName(), path.getName());
    assertEquals(stati[0].getPath(), statl[0].getPath());
  }

  private void testFileStatusAttr() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory
      Path path = new Path("/tmp/tmp-snap-test");
      DistributedFileSystem distributedFs = (DistributedFileSystem) FileSystem
          .get(path.toUri(), this.getProxiedFSConf());
      distributedFs.mkdirs(path);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertFalse(fs.getFileStatus(path).isSnapshotEnabled(),
          "Snapshot should be disallowed by default");
      // Allow snapshot
      distributedFs.allowSnapshot(path);
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled(),
          "Snapshot enabled bit is not set in FileStatus");
      // Disallow snapshot
      distributedFs.disallowSnapshot(path);
      // Check FileStatus
      assertFalse(fs.getFileStatus(path).isSnapshotEnabled(),
          "Snapshot enabled bit is not cleared in FileStatus");
      // Cleanup
      fs.delete(path, true);
      fs.close();
      distributedFs.close();
    }
  }

  private static void assertSameListing(FileSystem expected, FileSystem
      actual, Path p) throws IOException {
    // Consume all the entries from both iterators
    RemoteIterator<FileStatus> exIt = expected.listStatusIterator(p);
    List<FileStatus> exStatuses = new ArrayList<>();
    while (exIt.hasNext()) {
      exStatuses.add(exIt.next());
    }
    RemoteIterator<FileStatus> acIt = actual.listStatusIterator(p);
    List<FileStatus> acStatuses = new ArrayList<>();
    while (acIt.hasNext()) {
      acStatuses.add(acIt.next());
    }
    assertEquals(exStatuses.size(), acStatuses.size());
    for (int i = 0; i < exStatuses.size(); i++) {
      FileStatus expectedStatus = exStatuses.get(i);
      FileStatus actualStatus = acStatuses.get(i);
      // Path URIs are fully qualified, so compare just the path component
      assertEquals(expectedStatus.getPath().toUri().getPath(),
          actualStatus.getPath().toUri().getPath());
    }
  }

  private void testListStatusBatch() throws Exception {
    // LocalFileSystem writes checksum files next to the data files, which
    // show up when listing via LFS. This makes the listings not compare
    // properly.
    assumeFalse(isLocalFS());

    FileSystem proxyFs = FileSystem.get(getProxiedFSConf());
    Configuration conf = new Configuration();
    conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 2);
    FileSystem httpFs = getHttpFSFileSystem(conf);

    // Test an empty directory
    Path dir = new Path(getProxiedFSTestDir(), "dir");
    proxyFs.mkdirs(dir);
    assertSameListing(proxyFs, httpFs, dir);
    // Create and test in a loop
    for (int i = 0; i < 10; i++) {
      proxyFs.create(new Path(dir, "file" + i)).close();
      assertSameListing(proxyFs, httpFs, dir);
    }

    // Test for HDFS-12139
    Path dir1 = new Path(getProxiedFSTestDir(), "dir1");
    proxyFs.mkdirs(dir1);
    Path file1 = new Path(dir1, "file1");
    proxyFs.create(file1).close();

    RemoteIterator<FileStatus> si = proxyFs.listStatusIterator(dir1);
    FileStatus statusl = si.next();
    FileStatus status = proxyFs.getFileStatus(file1);
    assertEquals(file1.getName(), statusl.getPath().getName());
    assertEquals(status.getPath(), statusl.getPath());

    si = proxyFs.listStatusIterator(file1);
    statusl = si.next();
    assertEquals(file1.getName(), statusl.getPath().getName());
    assertEquals(status.getPath(), statusl.getPath());
  }

  private void testWorkingdirectory() throws Exception {
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    Path workingDir = fs.getWorkingDirectory();
    fs.close();

    fs = getHttpFSFileSystem();
    if (isLocalFS()) {
      fs.setWorkingDirectory(workingDir);
    }
    Path httpFSWorkingDir = fs.getWorkingDirectory();
    fs.close();
    assertEquals(httpFSWorkingDir.toUri().getPath(),
                        workingDir.toUri().getPath());

    fs = getHttpFSFileSystem();
    fs.setWorkingDirectory(new Path("/tmp"));
    workingDir = fs.getWorkingDirectory();
    assertEquals(workingDir.toUri().getPath(),
        new Path("/tmp").toUri().getPath());
    final FileSystem httpFs = getHttpFSFileSystem();
    LambdaTestUtils.intercept(IllegalArgumentException.class,
        "Invalid DFS directory name /foo:bar",
        () -> httpFs.setWorkingDirectory(new Path("/foo:bar")));
    fs.setWorkingDirectory(new Path("/bar"));
    workingDir = fs.getWorkingDirectory();
    httpFs.close();
    fs.close();
    assertEquals(workingDir.toUri().getPath(),
        new Path("/bar").toUri().getPath());
  }

  private void testTrashRoot() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());

      final Path rootDir = new Path("/");
      final Path fooPath = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(fooPath);
      os.write(1);
      os.close();

      Path trashPath = fs.getTrashRoot(rootDir);
      Path fooTrashPath = fs.getTrashRoot(fooPath);
      fs.close();

      fs = getHttpFSFileSystem();
      Path httpFSTrashPath = fs.getTrashRoot(rootDir);
      Path httpFSFooTrashPath = fs.getTrashRoot(fooPath);
      fs.close();

      assertEquals(trashPath.toUri().getPath(),
          httpFSTrashPath.toUri().getPath());
      assertEquals(fooTrashPath.toUri().getPath(),
          httpFSFooTrashPath.toUri().getPath());
      // trash path is related to USER, not path
      assertEquals(trashPath.toUri().getPath(),
          fooTrashPath.toUri().getPath());
    }
  }

  private void testMkdirs() throws Exception {
    Path path = new Path(getProxiedFSTestDir(), "foo");
    FileSystem fs = getHttpFSFileSystem();
    fs.mkdirs(path);
    fs.close();
    fs = FileSystem.get(getProxiedFSConf());
    assertTrue(fs.exists(path));
    fs.close();
  }

  private void testSetTimes() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      FileStatus status1 = fs.getFileStatus(path);
      fs.close();
      long at = status1.getAccessTime();
      long mt = status1.getModificationTime();

      fs = getHttpFSFileSystem();
      fs.setTimes(path, mt - 10, at - 20);
      fs.close();

      fs = FileSystem.get(getProxiedFSConf());
      status1 = fs.getFileStatus(path);
      fs.close();
      long atNew = status1.getAccessTime();
      long mtNew = status1.getModificationTime();
      assertEquals(mtNew, mt - 10);
      assertEquals(atNew, at - 20);
    }
  }

  protected void testSetPermission() throws Exception {
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    Path path = new Path(getProxiedFSTestDir(), "foodir");
    fs.mkdirs(path);

    fs = getHttpFSFileSystem();
    FsPermission permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE);
    fs.setPermission(path, permission1);
    fs.close();

    fs = FileSystem.get(getProxiedFSConf());
    FileStatus status1 = fs.getFileStatus(path);
    fs.close();
    FsPermission permission2 = status1.getPermission();
    assertEquals(permission2, permission1);

    //sticky bit
    fs = getHttpFSFileSystem();
    permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE, true);
    fs.setPermission(path, permission1);
    fs.close();

    fs = FileSystem.get(getProxiedFSConf());
    status1 = fs.getFileStatus(path);
    fs.close();
    permission2 = status1.getPermission();
    assertTrue(permission2.getStickyBit());
    assertEquals(permission2, permission1);
  }

  private void testSetOwner() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      fs.close();

      fs = getHttpFSFileSystem();
      String user = HadoopUsersConfTestHelper.getHadoopUsers()[1];
      String group = HadoopUsersConfTestHelper.getHadoopUserGroups(user)[0];
      fs.setOwner(path, user, group);
      fs.close();

      fs = FileSystem.get(getProxiedFSConf());
      FileStatus status1 = fs.getFileStatus(path);
      fs.close();
      assertEquals(status1.getOwner(), user);
      assertEquals(status1.getGroup(), group);
    }
  }

  private void testSetReplication() throws Exception {
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    Path path = new Path(getProxiedFSTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);
    os.close();
    fs.setReplication(path, (short) 2);
    fs.close();

    fs = getHttpFSFileSystem();
    fs.setReplication(path, (short) 1);
    fs.close();

    fs = FileSystem.get(getProxiedFSConf());
    FileStatus status1 = fs.getFileStatus(path);
    fs.close();
    assertEquals(status1.getReplication(), (short) 1);
  }

  private void testChecksum() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      FileChecksum hdfsChecksum = fs.getFileChecksum(path);
      fs.close();
      fs = getHttpFSFileSystem();
      FileChecksum httpChecksum = fs.getFileChecksum(path);
      fs.close();
      assertEquals(httpChecksum.getAlgorithmName(),
          hdfsChecksum.getAlgorithmName());
      assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength());
      assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes());
    }
  }

  private void testContentSummary() throws Exception {
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    Path path = new Path(getProxiedFSTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);
    os.close();
    ContentSummary hdfsContentSummary = fs.getContentSummary(path);
    fs.close();
    fs = getHttpFSFileSystem();
    ContentSummary httpContentSummary = fs.getContentSummary(path);
    fs.close();
    assertEquals(hdfsContentSummary.getDirectoryCount(),
        httpContentSummary.getDirectoryCount());
    assertEquals(hdfsContentSummary.getErasureCodingPolicy(),
        httpContentSummary.getErasureCodingPolicy());
    assertEquals(hdfsContentSummary.getFileCount(),
        httpContentSummary.getFileCount());
    assertEquals(hdfsContentSummary.getLength(),
        httpContentSummary.getLength());
    assertEquals(hdfsContentSummary.getQuota(), httpContentSummary.getQuota());
    assertEquals(hdfsContentSummary.getSpaceConsumed(),
        httpContentSummary.getSpaceConsumed());
    assertEquals(hdfsContentSummary.getSpaceQuota(),
        httpContentSummary.getSpaceQuota());
  }

  private void testQuotaUsage() throws Exception {
    if (isLocalFS()) {
      // LocalFS doesn't support setQuota so skip here
      return;
    }

    DistributedFileSystem dfs =
        (DistributedFileSystem) FileSystem.get(getProxiedFSConf());
    Path path = new Path(getProxiedFSTestDir(), "foo");
    dfs.mkdirs(path);
    dfs.setQuota(path, 20, 600 * 1024 * 1024);
    for (int i = 0; i < 10; i++) {
      dfs.createNewFile(new Path(path, "test_file_" + i));
    }
    FSDataOutputStream out = dfs.create(new Path(path, "test_file"));
    out.writeUTF("Hello World");
    out.close();

    dfs.setQuotaByStorageType(path, StorageType.SSD, 100000);
    dfs.setQuotaByStorageType(path, StorageType.DISK, 200000);

    QuotaUsage hdfsQuotaUsage = dfs.getQuotaUsage(path);
    dfs.close();
    FileSystem fs = getHttpFSFileSystem();
    QuotaUsage httpQuotaUsage = fs.getQuotaUsage(path);
    fs.close();
    assertEquals(hdfsQuotaUsage.getFileAndDirectoryCount(),
        httpQuotaUsage.getFileAndDirectoryCount());
    assertEquals(hdfsQuotaUsage.getQuota(), httpQuotaUsage.getQuota());
    assertEquals(hdfsQuotaUsage.getSpaceConsumed(),
        httpQuotaUsage.getSpaceConsumed());
    assertEquals(hdfsQuotaUsage.getSpaceQuota(),
        httpQuotaUsage.getSpaceQuota());
    assertEquals(hdfsQuotaUsage.getTypeQuota(StorageType.SSD),
        httpQuotaUsage.getTypeQuota(StorageType.SSD));
    assertEquals(hdfsQuotaUsage.getTypeQuota(StorageType.DISK),
        httpQuotaUsage.getTypeQuota(StorageType.DISK));
  }
  
  /** Set xattr */
  private void testSetXAttr() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      fs.close();
 
      final String name1 = "user.a1";
      final byte[] value1 = new byte[]{0x31, 0x32, 0x33};
      final String name2 = "user.a2";
      final byte[] value2 = new byte[]{0x41, 0x42, 0x43};
      final String name3 = "user.a3";
      final byte[] value3 = null;
      final String name4 = "trusted.a1";
      final byte[] value4 = new byte[]{0x31, 0x32, 0x33};
      final String name5 = "a1";
      fs = getHttpFSFileSystem();
      fs.setXAttr(path, name1, value1);
      fs.setXAttr(path, name2, value2);
      fs.setXAttr(path, name3, value3);
      fs.setXAttr(path, name4, value4);
      try {
        fs.setXAttr(path, name5, value1);
        fail("Set xAttr with incorrect name format should fail.");
      } catch (IOException e) {
      } catch (IllegalArgumentException e) {
      }
      fs.close();

      fs = FileSystem.get(getProxiedFSConf());
      Map<String, byte[]> xAttrs = fs.getXAttrs(path);
      fs.close();
      assertEquals(4, xAttrs.size());
      assertArrayEquals(value1, xAttrs.get(name1));
      assertArrayEquals(value2, xAttrs.get(name2));
      assertArrayEquals(new byte[0], xAttrs.get(name3));
      assertArrayEquals(value4, xAttrs.get(name4));
    }
  }

  /** Get xattrs */
  private void testGetXAttrs() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      fs.close();

      final String name1 = "user.a1";
      final byte[] value1 = new byte[]{0x31, 0x32, 0x33};
      final String name2 = "user.a2";
      final byte[] value2 = new byte[]{0x41, 0x42, 0x43};
      final String name3 = "user.a3";
      final byte[] value3 = null;
      final String name4 = "trusted.a1";
      final byte[] value4 = new byte[]{0x31, 0x32, 0x33};
      fs = FileSystem.get(getProxiedFSConf());
      fs.setXAttr(path, name1, value1);
      fs.setXAttr(path, name2, value2);
      fs.setXAttr(path, name3, value3);
      fs.setXAttr(path, name4, value4);
      fs.close();

      // Get xattrs with names parameter
      fs = getHttpFSFileSystem();
      List<String> names = Lists.newArrayList();
      names.add(name1);
      names.add(name2);
      names.add(name3);
      names.add(name4);
      Map<String, byte[]> xAttrs = fs.getXAttrs(path, names);
      fs.close();
      assertEquals(4, xAttrs.size());
      assertArrayEquals(value1, xAttrs.get(name1));
      assertArrayEquals(value2, xAttrs.get(name2));
      assertArrayEquals(new byte[0], xAttrs.get(name3));
      assertArrayEquals(value4, xAttrs.get(name4));

      // Get specific xattr
      fs = getHttpFSFileSystem();
      byte[] value = fs.getXAttr(path, name1);
      assertArrayEquals(value1, value);
      final String name5 = "a1";
      try {
        value = fs.getXAttr(path, name5);
        fail("Get xAttr with incorrect name format should fail.");
      } catch (IOException e) {
      } catch (IllegalArgumentException e) {
      }
      fs.close();

      // Get all xattrs
      fs = getHttpFSFileSystem();
      xAttrs = fs.getXAttrs(path);
      fs.close();
      assertEquals(4, xAttrs.size());
      assertArrayEquals(value1, xAttrs.get(name1));
      assertArrayEquals(value2, xAttrs.get(name2));
      assertArrayEquals(new byte[0], xAttrs.get(name3));
      assertArrayEquals(value4, xAttrs.get(name4));
    }
  }

  /** Remove xattr */
  private void testRemoveXAttr() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      fs.close();

      final String name1 = "user.a1";
      final byte[] value1 = new byte[]{0x31, 0x32, 0x33};
      final String name2 = "user.a2";
      final byte[] value2 = new byte[]{0x41, 0x42, 0x43};
      final String name3 = "user.a3";
      final byte[] value3 = null;
      final String name4 = "trusted.a1";
      final byte[] value4 = new byte[]{0x31, 0x32, 0x33};
      final String name5 = "a1";
      fs = FileSystem.get(getProxiedFSConf());
      fs.setXAttr(path, name1, value1);
      fs.setXAttr(path, name2, value2);
      fs.setXAttr(path, name3, value3);
      fs.setXAttr(path, name4, value4);
      fs.close();

      fs = getHttpFSFileSystem();
      fs.removeXAttr(path, name1);
      fs.removeXAttr(path, name3);
      fs.removeXAttr(path, name4);
      try {
        fs.removeXAttr(path, name5);
        fail("Remove xAttr with incorrect name format should fail.");
      } catch (IOException e) {
      } catch (IllegalArgumentException e) {
      }

      fs = FileSystem.get(getProxiedFSConf());
      Map<String, byte[]> xAttrs = fs.getXAttrs(path);
      fs.close();
      assertEquals(1, xAttrs.size());
      assertArrayEquals(value2, xAttrs.get(name2));
    }
  }

  /** List xattrs */
  private void testListXAttrs() throws Exception {
    if (!isLocalFS()) {
      FileSystem fs = FileSystem.get(getProxiedFSConf());
      fs.mkdirs(getProxiedFSTestDir());
      Path path = new Path(getProxiedFSTestDir(), "foo.txt");
      OutputStream os = fs.create(path);
      os.write(1);
      os.close();
      fs.close();

      final String name1 = "user.a1";
      final byte[] value1 = new byte[]{0x31, 0x32, 0x33};
      final String name2 = "user.a2";
      final byte[] value2 = new byte[]{0x41, 0x42, 0x43};
      final String name3 = "user.a3";
      final byte[] value3 = null;
      final String name4 = "trusted.a1";
      final byte[] value4 = new byte[]{0x31, 0x32, 0x33};
      fs = FileSystem.get(getProxiedFSConf());
      fs.setXAttr(path, name1, value1);
      fs.setXAttr(path, name2, value2);
      fs.setXAttr(path, name3, value3);
      fs.setXAttr(path, name4, value4);
      fs.close();

      fs = getHttpFSFileSystem();
      List<String> names = fs.listXAttrs(path);
      assertEquals(4, names.size());
      assertTrue(names.contains(name1));
      assertTrue(names.contains(name2));
      assertTrue(names.contains(name3));
      assertTrue(names.contains(name4));
    }
  }

  /**
   * Runs assertions testing that two AclStatus objects contain the same info
   * @param a First AclStatus
   * @param b Second AclStatus
   * @throws Exception
   */
  private void assertSameAcls(AclStatus a, AclStatus b) throws Exception {
    assertEquals(a.getOwner(), b.getOwner());
    assertEquals(a.getGroup(), b.getGroup());
    assertEquals(a.getPermission(), b.getPermission());
    assertEquals(a.isStickyBit(), b.isStickyBit());
    assertEquals(a.getEntries().size(), b.getEntries().size());
    for (AclEntry e : a.getEntries()) {
      assertTrue(b.getEntries().contains(e));
    }
    for (AclEntry e : b.getEntries()) {
      assertTrue(a.getEntries().contains(e));
    }
  }

  private static void assertSameAcls(FileSystem expected, FileSystem actual,
      Path path) throws IOException {
    FileStatus expectedFileStatus = expected.getFileStatus(path);
    FileStatus actualFileStatus = actual.getFileStatus(path);
    assertEquals(actualFileStatus.hasAcl(), expectedFileStatus.hasAcl());
    // backwards compat
    assertEquals(actualFileStatus.getPermission().getAclBit(),
        expectedFileStatus.getPermission().getAclBit());
  }

  /**
   * Simple ACL tests on a file:  Set an acl, add an acl, remove one acl,
   * and remove all acls.
   * @throws Exception
   */
  private void testFileAcls() throws Exception {
    if ( isLocalFS() ) {
      return;
    }

    final String aclUser1 = "user:foo:rw-";
    final String rmAclUser1 = "user:foo:";
    final String aclUser2 = "user:bar:r--";
    final String aclGroup1 = "group::r--";
    final String aclSet = "user::rwx," + aclUser1 + ","
            + aclGroup1 + ",other::---";

    FileSystem proxyFs = FileSystem.get(getProxiedFSConf());
    FileSystem httpfs = getHttpFSFileSystem();

    Path path = new Path(getProxiedFSTestDir(), "testAclStatus.txt");
    OutputStream os = proxyFs.create(path);
    os.write(1);
    os.close();

    AclStatus proxyAclStat = proxyFs.getAclStatus(path);
    AclStatus httpfsAclStat = httpfs.getAclStatus(path);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, path);

    httpfs.setAcl(path, AclEntry.parseAclSpec(aclSet,true));
    proxyAclStat = proxyFs.getAclStatus(path);
    httpfsAclStat = httpfs.getAclStatus(path);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, path);

    httpfs.modifyAclEntries(path, AclEntry.parseAclSpec(aclUser2, true));
    proxyAclStat = proxyFs.getAclStatus(path);
    httpfsAclStat = httpfs.getAclStatus(path);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, path);

    httpfs.removeAclEntries(path, AclEntry.parseAclSpec(rmAclUser1, false));
    proxyAclStat = proxyFs.getAclStatus(path);
    httpfsAclStat = httpfs.getAclStatus(path);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, path);

    httpfs.removeAcl(path);
    proxyAclStat = proxyFs.getAclStatus(path);
    httpfsAclStat = httpfs.getAclStatus(path);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, path);
  }

  /**
   * Simple acl tests on a directory: set a default acl, remove default acls.
   * @throws Exception
   */
  private void testDirAcls() throws Exception {
    if ( isLocalFS() ) {
      return;
    }

    final String defUser1 = "default:user:glarch:r-x";

    FileSystem proxyFs = FileSystem.get(getProxiedFSConf());
    FileSystem httpfs = getHttpFSFileSystem();

    Path dir = getProxiedFSTestDir();

    /* ACL Status on a directory */
    AclStatus proxyAclStat = proxyFs.getAclStatus(dir);
    AclStatus httpfsAclStat = httpfs.getAclStatus(dir);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, dir);

    /* Set a default ACL on the directory */
    httpfs.setAcl(dir, (AclEntry.parseAclSpec(defUser1,true)));
    proxyAclStat = proxyFs.getAclStatus(dir);
    httpfsAclStat = httpfs.getAclStatus(dir);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, dir);

    /* Remove the default ACL */
    httpfs.removeDefaultAcl(dir);
    proxyAclStat = proxyFs.getAclStatus(dir);
    httpfsAclStat = httpfs.getAclStatus(dir);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, dir);
  }

  private void testEncryption() throws Exception {
    if (isLocalFS()) {
      return;
    }
    FileSystem proxyFs = FileSystem.get(getProxiedFSConf());
    FileSystem httpFs = getHttpFSFileSystem();
    FileStatus proxyStatus = proxyFs.getFileStatus(TestHdfsHelper
        .ENCRYPTED_FILE);
    assertTrue(proxyStatus.isEncrypted());
    FileStatus httpStatus = httpFs.getFileStatus(TestHdfsHelper
        .ENCRYPTED_FILE);
    assertTrue(httpStatus.isEncrypted());
    proxyStatus = proxyFs.getFileStatus(new Path("/"));
    httpStatus = httpFs.getFileStatus(new Path("/"));
    assertFalse(proxyStatus.isEncrypted());
    assertFalse(httpStatus.isEncrypted());
  }

  private void testErasureCoding() throws Exception {
    assumeFalse(isLocalFS(), "Assume its not a local FS!");
    FileSystem proxyFs = FileSystem.get(getProxiedFSConf());
    FileSystem httpFS = getHttpFSFileSystem();
    Path filePath = new Path(getProxiedFSTestDir(), "foo.txt");
    proxyFs.create(filePath).close();

    ContractTestUtils.assertNotErasureCoded(httpFS, getProxiedFSTestDir());
    ContractTestUtils.assertNotErasureCoded(httpFS, filePath);
    ContractTestUtils.assertErasureCoded(httpFS,
        TestHdfsHelper.ERASURE_CODING_DIR);
    ContractTestUtils.assertErasureCoded(httpFS,
        TestHdfsHelper.ERASURE_CODING_FILE);

    proxyFs.close();
    httpFS.close();
  }

  private void testStoragePolicy() throws Exception {
    assumeFalse(isLocalFS(), "Assume its not a local FS");
    FileSystem fs = FileSystem.get(getProxiedFSConf());
    fs.mkdirs(getProxiedFSTestDir());
    Path path = new Path(getProxiedFSTestDir(), "policy.txt");
    FileSystem httpfs = getHttpFSFileSystem();
    // test getAllStoragePolicies
    assertArrayEquals(fs.getAllStoragePolicies().toArray(),
        httpfs.getAllStoragePolicies().toArray(),
        "Policy array returned from the DFS and HttpFS should be equals");

    // test get/set/unset policies
    DFSTestUtil.createFile(fs, path, 0, (short) 1, 0L);
    // get defaultPolicy
   BlockStoragePolicySpi defaultdfsPolicy = fs.getStoragePolicy(path);
    // set policy through webhdfs
    httpfs.setStoragePolicy(path, HdfsConstants.COLD_STORAGE_POLICY_NAME);
    // get policy from dfs
    BlockStoragePolicySpi dfsPolicy = fs.getStoragePolicy(path);
    // get policy from webhdfs
    BlockStoragePolicySpi httpFsPolicy = httpfs.getStoragePolicy(path);
    assertEquals(HdfsConstants.COLD_STORAGE_POLICY_NAME.toString(),
        httpFsPolicy.getName(), "Storage policy returned from the get API should" +
        " be same as set policy");
    assertEquals(httpFsPolicy, dfsPolicy,
        "Storage policy returned from the DFS and HttpFS should be equals");
    // unset policy
    httpfs.unsetStoragePolicy(path);
    assertEquals(defaultdfsPolicy, httpfs.getStoragePolicy(path),
        "After unset storage policy, the get API shoudld return the default policy");
    fs.close();
  }

  protected enum Operation {
    GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS,
    WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER,
    SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, QUOTA_USAGE, FILEACLS, DIRACLS,
    SET_XATTR, GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION,
    LIST_STATUS_BATCH, GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING,
    CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT,
    ALLOW_SNAPSHOT, DISALLOW_SNAPSHOT, DISALLOW_SNAPSHOT_EXCEPTION,
    FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST,
    GET_SNAPSHOT_LIST, GET_SERVERDEFAULTS, CHECKACCESS, SETECPOLICY,
    SATISFYSTORAGEPOLICY, GET_SNAPSHOT_DIFF_LISTING, GETFILEBLOCKLOCATIONS,
    GETFILELINKSTATUS, GETSTATUS, GETECPOLICIES, GETECCODECS, GETTRASHROOTS
  }
  @SuppressWarnings("methodlength")
  private void operation(Operation op) throws Exception {
    switch (op) {
    case GET:
      testGet();
      break;
    case OPEN:
      testOpen();
      break;
    case CREATE:
      testCreate();
      break;
    case APPEND:
      testAppend();
      break;
    case TRUNCATE:
      testTruncate();
      break;
    case CONCAT:
      testConcat();
      break;
    case RENAME:
      testRename();
      break;
    case DELETE:
      testDelete();
      break;
    case LIST_STATUS:
      testListStatus();
      testListSymLinkStatus();
      break;
    case WORKING_DIRECTORY:
      testWorkingdirectory();
      break;
    case MKDIRS:
      testMkdirs();
      break;
    case SET_TIMES:
      testSetTimes();
      break;
    case SET_PERMISSION:
      testSetPermission();
      break;
    case SET_OWNER:
      testSetOwner();
      break;
    case SET_REPLICATION:
      testSetReplication();
      break;
    case CHECKSUM:
      testChecksum();
      break;
    case CONTENT_SUMMARY:
      testContentSummary();
      break;
    case QUOTA_USAGE:
      testQuotaUsage();
      break;
    case FILEACLS:
      testFileAclsCustomizedUserAndGroupNames();
      testFileAcls();
      break;
    case DIRACLS:
      testDirAcls();
      break;
    case SET_XATTR:
      testSetXAttr();
      break;
    case REMOVE_XATTR:
      testRemoveXAttr();
      break;
    case GET_XATTRS:
      testGetXAttrs();
      break;
    case LIST_XATTRS:
      testListXAttrs();
      break;
    case ENCRYPTION:
      testEncryption();
      break;
    case LIST_STATUS_BATCH:
      testListStatusBatch();
      break;
    case GETTRASHROOT:
      testTrashRoot();
      break;
    case STORAGEPOLICY:
      testStoragePolicy();
      break;
    case ERASURE_CODING:
      testErasureCoding();
      break;
    case CREATE_SNAPSHOT:
      testCreateSnapshot();
      break;
    case RENAME_SNAPSHOT:
      testRenameSnapshot();
      break;
    case DELETE_SNAPSHOT:
      testDeleteSnapshot();
      break;
    case ALLOW_SNAPSHOT:
      testAllowSnapshot();
      break;
    case DISALLOW_SNAPSHOT:
      testDisallowSnapshot();
      break;
    case DISALLOW_SNAPSHOT_EXCEPTION:
      testDisallowSnapshotException();
      break;
    case FILE_STATUS_ATTR:
      testFileStatusAttr();
      break;
    case GET_SNAPSHOT_DIFF:
      testGetSnapshotDiff();
      testGetSnapshotDiffIllegalParam();
      break;
    case GET_SNAPSHOTTABLE_DIRECTORY_LIST:
      testGetSnapshottableDirListing();
      break;
    case GET_SNAPSHOT_LIST:
      testGetSnapshotListing();
      break;
    case GET_SERVERDEFAULTS:
      testGetServerDefaults();
      break;
    case CHECKACCESS:
      testAccess();
      break;
    case SETECPOLICY:
      testErasureCodingPolicy();
      break;
    case SATISFYSTORAGEPOLICY:
      testStoragePolicySatisfier();
      break;
    case GET_SNAPSHOT_DIFF_LISTING:
      testGetSnapshotDiffListing();
      break;
    case GETFILEBLOCKLOCATIONS:
      testGetFileBlockLocations();
      break;
    case GETFILELINKSTATUS:
      testGetFileLinkStatus();
      break;
    case GETSTATUS:
      testGetStatus();
      break;
    case GETECPOLICIES:
      testGetAllEEPolicies();
      break;
    case GETECCODECS:
      testGetECCodecs();
      break;
    case GETTRASHROOTS:
      testGetTrashRoots();
      break;
    }
  }

  public static Collection operations() {
    Object[][] ops = new Object[Operation.values().length][];
    for (int i = 0; i < Operation.values().length; i++) {
      ops[i] = new Object[]{Operation.values()[i]};
    }
    //To test one or a subset of operations do:
    //return Arrays.asList(new Object[][]{ new Object[]{Operation.APPEND}});
    return Arrays.asList(ops);
  }

  private Operation operation;

  public void initBaseTestHttpFSWith(Operation pOperation) {
    this.operation = pOperation;
  }

  @MethodSource("operations")
  @ParameterizedTest
  @TestDir
  @TestJetty
  @TestHdfs
  public void testOperation(Operation pOperation) throws Exception {
    initBaseTestHttpFSWith(pOperation);
    createHttpFSServer();
    operation(operation);
  }

  @MethodSource("operations")
  @ParameterizedTest
  @TestDir
  @TestJetty
  @TestHdfs
  public void testOperationDoAs(Operation pOperation) throws Exception {
    initBaseTestHttpFSWith(pOperation);
    createHttpFSServer();
    UserGroupInformation ugi = UserGroupInformation.createProxyUser(HadoopUsersConfTestHelper.getHadoopUsers()[0],
                                                                    UserGroupInformation.getCurrentUser());
    ugi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        operation(operation);
        return null;
      }
    });
  }

  private void testCreateSnapshot(String snapshotName) throws Exception {
    if (!this.isLocalFS()) {
      Path snapshottablePath = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(snapshottablePath);
      //Now get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      if (snapshotName == null) {
        fs.createSnapshot(snapshottablePath);
      } else {
        fs.createSnapshot(snapshottablePath, snapshotName);
      }
      Path snapshotsDir = new Path("/tmp/tmp-snap-test/.snapshot");
      FileStatus[] snapshotItems = fs.listStatus(snapshotsDir);
      assertTrue(snapshotItems.length == 1,
          "Should have exactly one snapshot.");
      String resultingSnapName = snapshotItems[0].getPath().getName();
      if (snapshotName == null) {
        assertTrue(Pattern.matches("(s)(\\d{8})(-)(\\d{6})(\\.)(\\d{3})",
            resultingSnapName), "Snapshot auto generated name not matching pattern");
      } else {
        assertTrue(snapshotName.equals(resultingSnapName),
            "Snapshot name is not same as passed name.");
      }
      cleanSnapshotTests(snapshottablePath, resultingSnapName);
    }
  }

  private void testCreateSnapshot() throws Exception {
    testCreateSnapshot(null);
    testCreateSnapshot("snap-with-name");
  }

  private void createSnapshotTestsPreconditions(Path snapshottablePath,
      Boolean allowSnapshot) throws Exception {
    //Needed to get a DistributedFileSystem instance, in order to
    //call allowSnapshot on the newly created directory
    DistributedFileSystem distributedFs = (DistributedFileSystem)
        FileSystem.get(snapshottablePath.toUri(), this.getProxiedFSConf());
    distributedFs.mkdirs(snapshottablePath);
    if (allowSnapshot) {
      distributedFs.allowSnapshot(snapshottablePath);
    }
    Path subdirPath = new Path("/tmp/tmp-snap-test/subdir");
    distributedFs.mkdirs(subdirPath);
  }

  private void createSnapshotTestsPreconditions(Path snapshottablePath)
      throws Exception {
    // Allow snapshot by default for snapshot test
    createSnapshotTestsPreconditions(snapshottablePath, true);
  }

  private void cleanSnapshotTests(Path snapshottablePath,
                                  String resultingSnapName) throws Exception {
    DistributedFileSystem distributedFs = (DistributedFileSystem)
        FileSystem.get(snapshottablePath.toUri(), this.getProxiedFSConf());
    distributedFs.deleteSnapshot(snapshottablePath, resultingSnapName);
    distributedFs.delete(snapshottablePath, true);
  }

  private void testRenameSnapshot() throws Exception {
    if (!this.isLocalFS()) {
      Path snapshottablePath = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(snapshottablePath);
      //Now get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      fs.createSnapshot(snapshottablePath, "snap-to-rename");
      fs.renameSnapshot(snapshottablePath, "snap-to-rename",
          "snap-new-name");
      Path snapshotsDir = new Path("/tmp/tmp-snap-test/.snapshot");
      FileStatus[] snapshotItems = fs.listStatus(snapshotsDir);
      assertTrue(snapshotItems.length == 1,
          "Should have exactly one snapshot.");
      String resultingSnapName = snapshotItems[0].getPath().getName();
      assertTrue("snap-new-name".equals(resultingSnapName),
          "Snapshot name is not same as passed name.");
      cleanSnapshotTests(snapshottablePath, resultingSnapName);
    }
  }

  private void testDeleteSnapshot() throws Exception {
    if (!this.isLocalFS()) {
      Path snapshottablePath = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(snapshottablePath);
      //Now get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      fs.createSnapshot(snapshottablePath, "snap-to-delete");
      Path snapshotsDir = new Path("/tmp/tmp-snap-test/.snapshot");
      FileStatus[] snapshotItems = fs.listStatus(snapshotsDir);
      assertTrue(snapshotItems.length == 1,
          "Should have exactly one snapshot.");
      fs.deleteSnapshot(snapshottablePath, "snap-to-delete");
      snapshotItems = fs.listStatus(snapshotsDir);
      assertTrue(snapshotItems.length == 0,
          "There should be no snapshot anymore.");
      fs.delete(snapshottablePath, true);
    }
  }

  private void testAllowSnapshot() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory with snapshot disallowed
      Path path = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(path, false);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertFalse(fs.getFileStatus(path).isSnapshotEnabled(),
          "Snapshot should be disallowed by default");
      // Allow snapshot
      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        httpFS.allowSnapshot(path);
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        webHdfsFileSystem.allowSnapshot(path);
      } else {
        fail(fs.getClass().getSimpleName() +
            " doesn't support allowSnapshot");
      }
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled(),
          "allowSnapshot failed");
      // Cleanup
      fs.delete(path, true);
    }
  }

  private void testDisallowSnapshot() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory with snapshot allowed
      Path path = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(path);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled(),
          "Snapshot should be allowed by DFS");
      // Disallow snapshot
      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        httpFS.disallowSnapshot(path);
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        webHdfsFileSystem.disallowSnapshot(path);
      } else {
        fail(fs.getClass().getSimpleName() +
            " doesn't support disallowSnapshot");
      }
      // Check FileStatus
      assertFalse(fs.getFileStatus(path).isSnapshotEnabled(),
          "disallowSnapshot failed");
      // Cleanup
      fs.delete(path, true);
    }
  }

  private void testDisallowSnapshotException() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory with snapshot allowed
      Path path = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(path);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled(),
          "Snapshot should be allowed by DFS");
      // Create some snapshots
      fs.createSnapshot(path, "snap-01");
      fs.createSnapshot(path, "snap-02");
      // Disallow snapshot
      boolean disallowSuccess = false;
      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        try {
          httpFS.disallowSnapshot(path);
          disallowSuccess = true;
        } catch (SnapshotException e) {
          // Expect SnapshotException
        }
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        try {
          webHdfsFileSystem.disallowSnapshot(path);
          disallowSuccess = true;
        } catch (SnapshotException e) {
          // Expect SnapshotException
        }
      } else {
        fail(fs.getClass().getSimpleName() +
            " doesn't support disallowSnapshot");
      }
      if (disallowSuccess) {
        fail("disallowSnapshot doesn't throw SnapshotException when "
            + "disallowing snapshot on a directory with at least one snapshot");
      }
      // Check FileStatus, should still be enabled since
      // disallow snapshot should fail
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled(),
          "disallowSnapshot should not have succeeded");
      // Cleanup
      fs.deleteSnapshot(path, "snap-02");
      fs.deleteSnapshot(path, "snap-01");
      fs.delete(path, true);
    }
  }

  private void testGetSnapshotDiff() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory with snapshot allowed
      Path path = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(path);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled());
      // Create a file and take a snapshot
      Path file1 = new Path(path, "file1");
      testCreate(file1, false);
      fs.createSnapshot(path, "snap1");
      // Create another file and take a snapshot
      Path file2 = new Path(path, "file2");
      testCreate(file2, false);
      fs.createSnapshot(path, "snap2");

      try {
        // Get snapshot diff
        SnapshotDiffReport diffReport = null;
        if (fs instanceof HttpFSFileSystem) {
          HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
          diffReport = httpFS.getSnapshotDiffReport(path, "snap1", "snap2");
        } else if (fs instanceof WebHdfsFileSystem) {
          WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
          diffReport = webHdfsFileSystem.getSnapshotDiffReport(path, "snap1", "snap2");
        } else {
          fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff");
        }
        // Verify result with DFS
        DistributedFileSystem dfs =
            (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
        SnapshotDiffReport dfsDiffReport = dfs.getSnapshotDiffReport(path, "snap1", "snap2");
        assertEquals(diffReport.toString(), dfsDiffReport.toString());
      } finally {
        // Cleanup
        fs.deleteSnapshot(path, "snap2");
        fs.deleteSnapshot(path, "snap1");
        fs.delete(path, true);
      }
    }
  }

  private void testGetSnapshotDiffIllegalParamCase(FileSystem fs, Path path,
      String oldsnapshotname, String snapshotname) throws IOException {
    try {
      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        httpFS.getSnapshotDiffReport(path, oldsnapshotname, snapshotname);
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        webHdfsFileSystem.getSnapshotDiffReport(path, oldsnapshotname,
            snapshotname);
      } else {
        fail(fs.getClass().getSimpleName() +
            " doesn't support getSnapshotDiff");
      }
    } catch (SnapshotException|IllegalArgumentException|RemoteException e) {
      // Expect SnapshotException, IllegalArgumentException
      // or RemoteException(IllegalArgumentException)
      if (e instanceof RemoteException) {
        // Check RemoteException class name, should be IllegalArgumentException
        assertEquals(((RemoteException) e).getClassName()
            .compareTo(java.lang.IllegalArgumentException.class.getName()), 0);
      }
      return;
    }
    fail("getSnapshotDiff illegal param didn't throw Exception");
  }

  private void testGetSnapshotDiffIllegalParam() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory with snapshot allowed
      Path path = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(path);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled(),
          "Snapshot should be allowed by DFS");
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled());
      // Get snapshot diff
      testGetSnapshotDiffIllegalParamCase(fs, path, "", "");
      testGetSnapshotDiffIllegalParamCase(fs, path, "snap1", "");
      testGetSnapshotDiffIllegalParamCase(fs, path, "", "snap2");
      testGetSnapshotDiffIllegalParamCase(fs, path, "snap1", "snap2");
      // Cleanup
      fs.delete(path, true);
    }
  }

  private void verifyGetSnapshottableDirListing(
      FileSystem fs, DistributedFileSystem dfs) throws Exception {
    // Get snapshottable directory list
    SnapshottableDirectoryStatus[] sds = null;
    if (fs instanceof HttpFSFileSystem) {
      HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
      sds = httpFS.getSnapshottableDirectoryList();
    } else if (fs instanceof WebHdfsFileSystem) {
      WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
      sds = webHdfsFileSystem.getSnapshottableDirectoryList();
    } else {
      fail(fs.getClass().getSimpleName() +
          " doesn't support getSnapshottableDirListing");
    }
    // Verify result with DFS
    SnapshottableDirectoryStatus[] dfssds = dfs.getSnapshottableDirListing();
    assertEquals(JsonUtil.toJsonString(sds),
        JsonUtil.toJsonString(dfssds));
  }

  private void testGetSnapshotListing() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory with snapshot allowed
      Path path = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(path);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled());
      // Create a file and take a snapshot
      Path file1 = new Path(path, "file1");
      testCreate(file1, false);
      fs.createSnapshot(path, "snap1");
      // Create another file and take a snapshot
      Path file2 = new Path(path, "file2");
      testCreate(file2, false);
      fs.createSnapshot(path, "snap2");
      // Get snapshot diff
      SnapshotStatus[] snapshotStatus = null;
      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        snapshotStatus = httpFS.getSnapshotListing(path);
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        snapshotStatus = webHdfsFileSystem.getSnapshotListing(path);
      } else {
        fail(fs.getClass().getSimpleName() +
            " doesn't support getSnapshotDiff");
      }
      // Verify result with DFS
      DistributedFileSystem dfs = (DistributedFileSystem)
          FileSystem.get(path.toUri(), this.getProxiedFSConf());
      SnapshotStatus[] dfsStatus =
          dfs.getSnapshotListing(path);
      assertEquals(JsonUtil.toJsonString(snapshotStatus),
          JsonUtil.toJsonString(dfsStatus));
      // Cleanup
      fs.deleteSnapshot(path, "snap2");
      fs.deleteSnapshot(path, "snap1");
      fs.delete(path, true);
    }
  }


  private void testGetSnapshottableDirListing() throws Exception {
    if (!this.isLocalFS()) {
      FileSystem fs = this.getHttpFSFileSystem();
      // Create directories with snapshot allowed
      Path path1 = new Path("/tmp/tmp-snap-dirlist-test-1");
      DistributedFileSystem dfs = (DistributedFileSystem)
          FileSystem.get(path1.toUri(), this.getProxiedFSConf());
      // Verify response when there is no snapshottable directory
      verifyGetSnapshottableDirListing(fs, dfs);
      createSnapshotTestsPreconditions(path1);
      assertTrue(fs.getFileStatus(path1).isSnapshotEnabled());
      // Verify response when there is one snapshottable directory
      verifyGetSnapshottableDirListing(fs, dfs);
      Path path2 = new Path("/tmp/tmp-snap-dirlist-test-2");
      createSnapshotTestsPreconditions(path2);
      assertTrue(fs.getFileStatus(path2).isSnapshotEnabled());
      // Verify response when there are two snapshottable directories
      verifyGetSnapshottableDirListing(fs, dfs);

      // Clean up and verify
      fs.delete(path2, true);
      verifyGetSnapshottableDirListing(fs, dfs);
      fs.delete(path1, true);
      verifyGetSnapshottableDirListing(fs, dfs);
    }
  }

  private void testFileAclsCustomizedUserAndGroupNames() throws Exception {
    if (isLocalFS()) {
      return;
    }

    // Get appropriate conf from the cluster
    MiniDFSCluster miniDFSCluster = ((TestHdfsHelper) hdfsTestHelper)
        .getMiniDFSCluster();
    Configuration conf = miniDFSCluster.getConfiguration(0);
    // If we call getHttpFSFileSystem() without conf from the mini cluster,
    // WebHDFS will be initialized with the default ACL string, causing the
    // setAcl() later to fail. This is only an issue in the unit test.
    FileSystem httpfs = getHttpFSFileSystem(conf);
    if (!(httpfs instanceof WebHdfsFileSystem)
        && !(httpfs instanceof HttpFSFileSystem)) {
      fail(httpfs.getClass().getSimpleName() +
          " doesn't support custom user and group name pattern. "
          + "Only WebHdfsFileSystem and HttpFSFileSystem support it.");
    }
    final String aclUser = "user:123:rwx";
    final String aclGroup = "group:foo@bar:r--";
    final String aclSet = "user::rwx," + aclUser + ",group::r--," +
        aclGroup + ",other::r--";
    final String dir = "/aclFileTestCustom";
    // Create test file
    FileSystem proxyFs = FileSystem.get(conf);
    proxyFs.mkdirs(new Path(dir));
    Path path = new Path(dir, "/testACL");
    OutputStream os = proxyFs.create(path);
    os.write(1);
    os.close();
    // Set ACL
    httpfs.setAcl(path, AclEntry.parseAclSpec(aclSet, true));
    // Verify getAclStatus responses are the same
    AclStatus proxyAclStat = proxyFs.getAclStatus(path);
    AclStatus httpfsAclStat = httpfs.getAclStatus(path);
    assertSameAcls(httpfsAclStat, proxyAclStat);
    assertSameAcls(httpfs, proxyFs, path);
    // Verify that custom user and group are set.
    List<String> strEntries = new ArrayList<>();
    for (AclEntry aclEntry : httpfsAclStat.getEntries()) {
      strEntries.add(aclEntry.toStringStable());
    }
    assertTrue(strEntries.contains(aclUser));
    assertTrue(strEntries.contains(aclGroup));
    // Clean up
    proxyFs.delete(new Path(dir), true);
  }

  private void verifyGetServerDefaults(FileSystem fs, DistributedFileSystem dfs)
      throws Exception {
    FsServerDefaults sds = null;
    if (fs instanceof HttpFSFileSystem) {
      HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
      sds = httpFS.getServerDefaults();
    } else if (fs instanceof WebHdfsFileSystem) {
      WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
      sds = webHdfsFileSystem.getServerDefaults();
    } else {
      fail(fs.getClass().getSimpleName() + " doesn't support getServerDefaults");
    }
    // Verify result with DFS
    FsServerDefaults dfssds = dfs.getServerDefaults();
    assertEquals(JsonUtil.toJsonString(sds),
        JsonUtil.toJsonString(dfssds));
  }

  private void testGetServerDefaults() throws Exception {
    if (!this.isLocalFS()) {
      FileSystem fs = this.getHttpFSFileSystem();
      Path path1 = new Path("/");
      DistributedFileSystem dfs = (DistributedFileSystem) FileSystem
          .get(path1.toUri(), this.getProxiedFSConf());
      verifyGetServerDefaults(fs, dfs);
    }
  }

  private void testAccess() throws Exception {
    if (!this.isLocalFS()) {
      FileSystem fs = this.getHttpFSFileSystem();
      Path path1 = new Path("/");
      DistributedFileSystem dfs = (DistributedFileSystem) FileSystem
          .get(path1.toUri(), this.getProxiedFSConf());
      verifyAccess(fs, dfs);
    }
  }

  private void verifyAccess(FileSystem fs, DistributedFileSystem dfs)
      throws Exception {
    Path p1 = new Path("/p1");
    dfs.mkdirs(p1);
    dfs.setOwner(p1, "user1", "group1");
    dfs.setPermission(p1, new FsPermission((short) 0444));

    if (fs instanceof HttpFSFileSystem) {
      HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
      httpFS.access(p1, FsAction.READ);
    } else if (fs instanceof WebHdfsFileSystem) {
      WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
      webHdfsFileSystem.access(p1, FsAction.READ);
    } else {
      fail(fs.getClass().getSimpleName() + " doesn't support access");
    }
  }

  private void testErasureCodingPolicy() throws Exception {
    if (!this.isLocalFS()) {
      FileSystem fs = this.getHttpFSFileSystem();
      Path path1 = new Path("/");
      DistributedFileSystem dfs = (DistributedFileSystem) FileSystem
          .get(path1.toUri(), this.getProxiedFSConf());
      final String dir = "/xattrTest";
      Path p1 = new Path(dir);

      final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies
          .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
      final String ecPolicyName = ecPolicy.getName();
      dfs.mkdirs(new Path(dir));
      dfs.enableErasureCodingPolicy(ecPolicyName);

      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        httpFS.setErasureCodingPolicy(p1, ecPolicyName);
        ErasureCodingPolicy ecPolicy1 = httpFS.getErasureCodingPolicy(p1);
        assertEquals(ecPolicy, ecPolicy1);
        httpFS.unsetErasureCodingPolicy(p1);
        ecPolicy1 = httpFS.getErasureCodingPolicy(p1);
        assertNull(ecPolicy1);
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        webHdfsFileSystem.setErasureCodingPolicy(p1, ecPolicyName);
        ErasureCodingPolicy ecPolicy1 =
            webHdfsFileSystem.getErasureCodingPolicy(p1);
        assertEquals(ecPolicy, ecPolicy1);
        webHdfsFileSystem.unsetErasureCodingPolicy(p1);
        ecPolicy1 = dfs.getErasureCodingPolicy(p1);
        assertNull(ecPolicy1);
      } else {
        fail(fs.getClass().getSimpleName() + " doesn't support access");
      }
    }
  }

  public void testStoragePolicySatisfier() throws Exception {
    final String dir = "/parent";
    Path path1 = new Path(dir);
    String file = "/parent/file";
    Path filePath = new Path(file);
    if (!this.isLocalFS()) {
      FileSystem fs = this.getHttpFSFileSystem();
      DistributedFileSystem dfs = (DistributedFileSystem) FileSystem
          .get(path1.toUri(), this.getProxiedFSConf());
      dfs.mkdirs(path1);
      dfs.create(filePath).close();
      dfs.setStoragePolicy(filePath, HdfsConstants.COLD_STORAGE_POLICY_NAME);
      BlockStoragePolicy storagePolicy =
          (BlockStoragePolicy) dfs.getStoragePolicy(filePath);
      assertEquals(HdfsConstants.COLD_STORAGE_POLICY_NAME,
          storagePolicy.getName());
      Map<String, byte[]> xAttrs;
      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        httpFS.satisfyStoragePolicy(path1);
        xAttrs = httpFS.getXAttrs(path1);
        assertTrue(xAttrs
            .containsKey(HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY));
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        webHdfsFileSystem.satisfyStoragePolicy(path1);
        xAttrs = webHdfsFileSystem.getXAttrs(path1);
        assertTrue(xAttrs
            .containsKey(HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY));
      } else {
        fail(fs.getClass().getSimpleName() + " doesn't support access");
      }
      dfs.delete(path1, true);
    }
  }

  private void testGetFileBlockLocations() throws Exception {
    BlockLocation[] blockLocations;
    Path testFile;
    if (!this.isLocalFS()) {
      FileSystem fs = this.getHttpFSFileSystem();
      testFile = new Path(getProxiedFSTestDir(), "singleBlock.txt");
      DFSTestUtil.createFile(fs, testFile, 1, (short) 1, 0L);
      if (fs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
        blockLocations = httpFS.getFileBlockLocations(testFile, 0, 1);
        assertNotNull(blockLocations);

        // verify HttpFSFileSystem.toBlockLocations()
        String jsonString = JsonUtil.toJsonString(blockLocations);
        JSONParser parser = new JSONParser();
        JSONObject jsonObject = (JSONObject) parser.parse(jsonString, (ContainerFactory) null);
        BlockLocation[] deserializedLocation = HttpFSFileSystem.toBlockLocations(jsonObject);
        assertEquals(blockLocations.length, deserializedLocation.length);
        for (int i = 0; i < blockLocations.length; i++) {
          assertEquals(blockLocations[i].toString(), deserializedLocation[i].toString());
        }
      } else if (fs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
        blockLocations = webHdfsFileSystem.getFileBlockLocations(testFile, 0, 1);
        assertNotNull(blockLocations);
      } else {
        fail(fs.getClass().getSimpleName() + " doesn't support access");
      }
    }
  }

  private void testGetSnapshotDiffListing() throws Exception {
    if (!this.isLocalFS()) {
      // Create a directory with snapshot allowed
      Path path = new Path("/tmp/tmp-snap-test");
      createSnapshotTestsPreconditions(path);
      // Get the FileSystem instance that's being tested
      FileSystem fs = this.getHttpFSFileSystem();
      // Check FileStatus
      assertTrue(fs.getFileStatus(path).isSnapshotEnabled());
      // Create a file and take a snapshot
      Path file1 = new Path(path, "file1");
      testCreate(file1, false);
      fs.createSnapshot(path, "snap1");
      // Create another file and take a snapshot
      Path file2 = new Path(path, "file2");
      testCreate(file2, false);
      fs.createSnapshot(path, "snap2");
      // Get snapshot diff listing
      try {
        SnapshotDiffReportListing diffReportListing = null;
        byte[] emptyBytes = new byte[] {};
        if (fs instanceof HttpFSFileSystem) {
          HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
          diffReportListing =
              httpFS.getSnapshotDiffReportListing(path, "snap1", "snap2", emptyBytes, -1);
        } else if (fs instanceof WebHdfsFileSystem) {
          WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
          diffReportListing = webHdfsFileSystem
              .getSnapshotDiffReportListing(path.toUri().getPath(), "snap1", "snap2", emptyBytes,
                  -1);
        } else {
          fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff");
        }
        // Verify result with DFS
        DistributedFileSystem dfs =
            (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
        SnapshotDiffReportListing dfsDiffReportListing =
            dfs.getSnapshotDiffReportListing(path, "snap1", "snap2",
                DFSUtil.bytes2String(emptyBytes), -1);
        assertHttpFsReportListingWithDfsClient(diffReportListing, dfsDiffReportListing);
      } finally {
        // Cleanup
        fs.deleteSnapshot(path, "snap2");
        fs.deleteSnapshot(path, "snap1");
        fs.delete(path, true);
      }
    }
  }

  private void testGetFileLinkStatus() throws Exception {
    if (isLocalFS()) {
      // do not test the symlink for local FS.
      return;
    }
    FileSystem fs = FileSystem.get(getProxiedFSConf());

    Path root = new Path(getProxiedFSTestDir(), "httpFSTest");
    Path file = new Path(root, "file");
    Path linkToFile = new Path(root, "linkToFile");

    OutputStream os = fs.create(file);
    os.write(1);
    fs.createSymlink(file, linkToFile, false);

    fs = this.getHttpFSFileSystem();

    assertFalse(fs.getFileLinkStatus(file).isSymlink());
    assertTrue(fs.getFileLinkStatus(linkToFile).isSymlink());
  }

  private void testGetStatus() throws Exception {
    if (isLocalFS()) {
      // do not test the getStatus for local FS.
      return;
    }
    final Path path = new Path("/foo");
    FileSystem fs = FileSystem.get(path.toUri(), this.getProxiedFSConf());
    if (fs instanceof DistributedFileSystem) {
      DistributedFileSystem dfs =
          (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
      FileSystem httpFs = this.getHttpFSFileSystem();

      FsStatus dfsFsStatus = dfs.getStatus(path);
      FsStatus httpFsStatus = httpFs.getStatus(path);

      //Validate used free and capacity are the same as DistributedFileSystem
      assertEquals(dfsFsStatus.getUsed(), httpFsStatus.getUsed());
      assertEquals(dfsFsStatus.getRemaining(), httpFsStatus.getRemaining());
      assertEquals(dfsFsStatus.getCapacity(), httpFsStatus.getCapacity());
      httpFs.close();
      dfs.close();
    } else {
      fail(fs.getClass().getSimpleName() + " is not of type DistributedFileSystem.");
    }
  }

  private void testGetAllEEPolicies() throws Exception {
    if (isLocalFS()) {
      // do not test the getAllEEPolicies for local FS.
      return;
    }
    final Path path = new Path("/foo");
    FileSystem fs = FileSystem.get(path.toUri(), this.getProxiedFSConf());
    if (fs instanceof DistributedFileSystem) {
      DistributedFileSystem dfs =
          (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
      FileSystem httpFs = this.getHttpFSFileSystem();

      Collection<ErasureCodingPolicyInfo> dfsAllErasureCodingPolicies =
          dfs.getAllErasureCodingPolicies();
      Collection<ErasureCodingPolicyInfo> diffErasureCodingPolicies = null;

      if (httpFs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) httpFs;
        diffErasureCodingPolicies = httpFS.getAllErasureCodingPolicies();
      } else if (httpFs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) httpFs;
        diffErasureCodingPolicies = webHdfsFileSystem.getAllErasureCodingPolicies();
      } else {
        fail(fs.getClass().getSimpleName() +
            " is not of type HttpFSFileSystem or WebHdfsFileSystem");
      }

      //Validate erasureCodingPolicyInfos are the same as DistributedFileSystem
      assertEquals(dfsAllErasureCodingPolicies.size(), diffErasureCodingPolicies.size());
      assertTrue(dfsAllErasureCodingPolicies.containsAll(diffErasureCodingPolicies));
    } else {
      fail(fs.getClass().getSimpleName() + " is not of type DistributedFileSystem.");
    }
  }

  private void testGetECCodecs() throws Exception {
    if (isLocalFS()) {
      // do not test the testGetECCodecs for local FS.
      return;
    }
    final Path path = new Path("/foo");

    FileSystem fs = FileSystem.get(path.toUri(), this.getProxiedFSConf());
    LambdaTestUtils.intercept(AssertionError.class, () -> {
      if (!(fs instanceof DistributedFileSystem)) {
        throw new AssertionError(fs.getClass().getSimpleName() +
            " is not of type DistributedFileSystem.");
      }
    });

    DistributedFileSystem dfs =
        (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
    FileSystem httpFs = this.getHttpFSFileSystem();

    Map<String, String> dfsErasureCodingCodecs = dfs.getAllErasureCodingCodecs();

    final AtomicReference<Map<String, String>> diffErasureCodingCodecsRef =
        new AtomicReference<>();
    LambdaTestUtils.intercept(AssertionError.class, () -> {
      if (httpFs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFSFileSystem = (HttpFSFileSystem) httpFs;
        diffErasureCodingCodecsRef.set(httpFSFileSystem.getAllErasureCodingCodecs());
      } else if (httpFs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) httpFs;
        diffErasureCodingCodecsRef.set(webHdfsFileSystem.getAllErasureCodingCodecs());
      } else {
        throw new AssertionError(httpFs.getClass().getSimpleName() +
            " is not of type HttpFSFileSystem or WebHdfsFileSystem");
      }
    });
    Map<String, String> diffErasureCodingCodecs = diffErasureCodingCodecsRef.get();

    //Validate testGetECCodecs are the same as DistributedFileSystem
    assertEquals(dfsErasureCodingCodecs.size(), diffErasureCodingCodecs.size());

    for (Map.Entry<String, String> entry : dfsErasureCodingCodecs.entrySet()) {
      String key = entry.getKey();
      String value = entry.getValue();
      assertTrue(diffErasureCodingCodecs.containsKey(key));
      assertEquals(value, diffErasureCodingCodecs.get(key));
    }
  }

  private void testGetTrashRoots() throws Exception {
    if (isLocalFS()) {
      // do not test the getAllEEPolicies for local FS.
      return;
    }
    final Path path = new Path("/");
    FileSystem fs = FileSystem.get(path.toUri(), this.getProxiedFSConf());
    if (fs instanceof DistributedFileSystem) {
      DistributedFileSystem dfs =
          (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
      FileSystem httpFs = this.getHttpFSFileSystem();

      // Create trash root for user0
      UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user0");
      String user0HomeStr = DFSUtilClient.getHomeDirectory(this.getProxiedFSConf(), ugi);
      Path user0Trash = new Path(user0HomeStr, FileSystem.TRASH_PREFIX);
      dfs.mkdirs(user0Trash);

      Collection<FileStatus> dfsTrashRoots = dfs.getTrashRoots(true);
      Collection<FileStatus> diffTrashRoots = null;

      if (httpFs instanceof HttpFSFileSystem) {
        HttpFSFileSystem httpFS = (HttpFSFileSystem) httpFs;
        diffTrashRoots = httpFS.getTrashRoots(true);
      } else if (httpFs instanceof WebHdfsFileSystem) {
        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) httpFs;
        diffTrashRoots = webHdfsFileSystem.getTrashRoots(true);
      } else {
        fail(fs.getClass().getSimpleName() +
            " is not of type HttpFSFileSystem or WebHdfsFileSystem");
      }

      // Validate getTrashRoots are the same as DistributedFileSystem
      assertEquals(dfsTrashRoots.size(), diffTrashRoots.size());
    } else {
      fail(fs.getClass().getSimpleName() + " is not of type DistributedFileSystem.");
    }
  }

  private void assertHttpFsReportListingWithDfsClient(SnapshotDiffReportListing diffReportListing,
      SnapshotDiffReportListing dfsDiffReportListing) {
    assertEquals(diffReportListing.getCreateList().size(),
        dfsDiffReportListing.getCreateList().size());
    assertEquals(diffReportListing.getDeleteList().size(),
        dfsDiffReportListing.getDeleteList().size());
    assertEquals(diffReportListing.getModifyList().size(),
        dfsDiffReportListing.getModifyList().size());
    assertEquals(diffReportListing.getIsFromEarlier(),
        dfsDiffReportListing.getIsFromEarlier());
    assertEquals(diffReportListing.getLastIndex(), dfsDiffReportListing.getLastIndex());
    assertEquals(DFSUtil.bytes2String(diffReportListing.getLastPath()),
        DFSUtil.bytes2String(dfsDiffReportListing.getLastPath()));
    int i = 0;
    for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing
        .getCreateList()) {
      SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry =
          dfsDiffReportListing.getCreateList().get(i);
      assertEquals(entry.getDirId(), dfsDiffEntry.getDirId());
      assertEquals(entry.getFileId(), dfsDiffEntry.getFileId());
      assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()),
          DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath()));
      i++;
    }
    i = 0;
    for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing
        .getDeleteList()) {
      SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry =
          dfsDiffReportListing.getDeleteList().get(i);
      assertEquals(entry.getDirId(), dfsDiffEntry.getDirId());
      assertEquals(entry.getFileId(), dfsDiffEntry.getFileId());
      assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()),
          DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath()));
      i++;
    }
    i = 0;
    for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing
        .getModifyList()) {
      SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry =
          dfsDiffReportListing.getModifyList().get(i);
      assertEquals(entry.getDirId(), dfsDiffEntry.getDirId());
      assertEquals(entry.getFileId(), dfsDiffEntry.getFileId());
      assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()),
          DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath()));
      i++;
    }
  }

}