public final class FileUtils extends Object
Modifier and Type | Field and Description |
---|---|
static org.apache.hadoop.fs.PathFilter |
HIDDEN_FILES_PATH_FILTER |
static int |
IO_ERROR_SLEEP_TIME |
static org.apache.hadoop.fs.PathFilter |
STAGING_DIR_PATH_FILTER |
Modifier and Type | Method and Description |
---|---|
static void |
checkDeletePermission(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration conf,
String user)
Checks if delete can be performed on given path by given user.
|
static void |
checkFileAccessWithImpersonation(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus stat,
org.apache.hadoop.fs.permission.FsAction action,
String user) |
static void |
checkFileAccessWithImpersonation(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus stat,
org.apache.hadoop.fs.permission.FsAction action,
String user,
List<org.apache.hadoop.fs.FileStatus> children,
org.apache.hadoop.fs.FileSystem fsAsUser)
Perform a check to determine if the user is able to access the file passed in.
|
static boolean |
checkIsOwnerOfFileHierarchy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus fileStatus,
String userName,
boolean recurse) |
static void |
closeFs(org.apache.hadoop.security.UserGroupInformation proxyUser) |
static boolean |
copy(org.apache.hadoop.fs.FileSystem srcFS,
org.apache.hadoop.fs.FileStatus srcStatus,
org.apache.hadoop.fs.FileSystem dstFS,
org.apache.hadoop.fs.Path dst,
boolean deleteSource,
boolean overwrite,
boolean preserveXAttrs,
org.apache.hadoop.conf.Configuration conf) |
static boolean |
copy(org.apache.hadoop.fs.FileSystem srcFS,
org.apache.hadoop.fs.Path[] srcs,
org.apache.hadoop.fs.FileSystem dstFS,
org.apache.hadoop.fs.Path dst,
boolean deleteSource,
boolean overwrite,
boolean preserveXAttr,
org.apache.hadoop.conf.Configuration conf) |
static boolean |
copy(org.apache.hadoop.fs.FileSystem srcFS,
org.apache.hadoop.fs.Path src,
org.apache.hadoop.fs.FileSystem dstFS,
org.apache.hadoop.fs.Path dst,
boolean deleteSource,
boolean overwrite,
HiveConf conf)
Copies files between filesystems.
|
static File |
createLocalDirsTempFile(org.apache.hadoop.conf.Configuration conf,
String prefix,
String suffix,
boolean isDirectory) |
static File |
createLocalDirsTempFile(String localDirList,
String prefix,
String suffix,
boolean isDirectory) |
static File |
createTempFile(String lScratchDir,
String prefix,
String suffix)
create temporary file and register it to delete-on-exit hook.
|
static void |
deleteDirectory(File directory) |
static void |
deleteIfExists(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
Attempts to delete a file if it exists.
|
static boolean |
deleteTmpFile(File tempFile)
delete a temporary file and remove it from delete-on-exit hook.
|
static boolean |
distCp(org.apache.hadoop.fs.FileSystem srcFS,
List<org.apache.hadoop.fs.Path> srcPaths,
org.apache.hadoop.fs.Path dst,
boolean deleteSource,
org.apache.hadoop.security.UserGroupInformation proxyUser,
HiveConf conf,
HadoopShims shims) |
static boolean |
distCpWithSnapshot(String oldSnapshot,
String newSnapshot,
List<org.apache.hadoop.fs.Path> srcPaths,
org.apache.hadoop.fs.Path dst,
boolean overwriteTarget,
HiveConf conf,
HadoopShims shims,
org.apache.hadoop.security.UserGroupInformation proxyUser) |
static boolean |
equalsFileSystem(org.apache.hadoop.fs.FileSystem fs1,
org.apache.hadoop.fs.FileSystem fs2) |
static String |
escapePathName(String path) |
static String |
escapePathName(String path,
String defaultPath)
Escapes a path name.
|
static org.apache.hadoop.fs.FileStatus |
getFileStatusOrNull(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
Attempts to get file status.
|
static org.apache.hadoop.fs.FileSystem |
getFsAsUser(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.security.UserGroupInformation proxyUser) |
static Set<String> |
getJarFilesByPath(String pathString,
org.apache.hadoop.conf.Configuration conf)
Given a path string, get all the jars from the folder or the files themselves.
|
static org.apache.hadoop.fs.Path |
getParentRegardlessOfScheme(org.apache.hadoop.fs.Path path,
Collection<org.apache.hadoop.fs.Path> candidates)
Returns the deepest candidate path for the given path.
|
static org.apache.hadoop.fs.FileStatus |
getPathOrParentThatExists(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
Find the parent of path that exists, if path does not exist
|
static org.apache.hadoop.security.UserGroupInformation |
getProxyUser(String user) |
static int |
getSleepTime(int repeatNum)
Returns the incremented sleep time in milli seconds.
|
static URI |
getURI(String path)
Get the URI of the path.
|
static boolean |
isActionPermittedForFileHierarchy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus fileStatus,
String userName,
org.apache.hadoop.fs.permission.FsAction action)
Check if user userName has permissions to perform the given FsAction action
on all files under the file whose FileStatus fileStatus is provided
|
static boolean |
isActionPermittedForFileHierarchy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus fileStatus,
String userName,
org.apache.hadoop.fs.permission.FsAction action,
boolean recurse) |
static boolean |
isActionPermittedForFileHierarchy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus fileStatus,
String userName,
org.apache.hadoop.fs.permission.FsAction action,
boolean recurse,
org.apache.hadoop.fs.FileSystem fsAsUser) |
static boolean |
isLocalFile(HiveConf conf,
String fileName)
A best effort attempt to determine if if the file is a local file
|
static boolean |
isLocalFile(HiveConf conf,
URI fileUri)
A best effort attempt to determine if if the file is a local file
|
static boolean |
isOwnerOfFileHierarchy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus fileStatus,
String userName) |
static boolean |
isOwnerOfFileHierarchy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus fileStatus,
String userName,
boolean recurse) |
static boolean |
isPathWithinSubtree(org.apache.hadoop.fs.Path path,
org.apache.hadoop.fs.Path subtree)
Checks whenever path is inside the given subtree
return true iff
* path = subtree
* subtreeContains(path,d) for any descendant of the subtree node
|
static boolean |
isS3a(org.apache.hadoop.fs.FileSystem fs) |
static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> |
listFiles(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
boolean recursive,
org.apache.hadoop.fs.PathFilter filter) |
static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> |
listStatusIterator(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
org.apache.hadoop.fs.PathFilter filter) |
static void |
listStatusRecursively(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus fileStatus,
List<org.apache.hadoop.fs.FileStatus> results)
Recursively lists status for all files starting from a particular directory (or individual file
as base case).
|
static org.apache.hadoop.fs.Path |
makeAbsolute(org.apache.hadoop.fs.FileSystem fileSystem,
org.apache.hadoop.fs.Path path) |
static String |
makeDefaultListBucketingDirName(List<String> skewedCols,
String name)
default directory will have the same depth as number of skewed columns
this will make future operation easy like DML merge, concatenate merge
|
static String |
makeListBucketingDirName(List<String> lbCols,
List<String> vals)
Makes a valid list bucketing directory name.
|
static String |
makePartName(List<String> partCols,
List<String> vals) |
static String |
makePartName(List<String> partCols,
List<String> vals,
String defaultStr)
Makes a valid partition name.
|
static org.apache.hadoop.fs.Path |
makeQualified(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration conf)
Variant of Path.makeQualified that qualifies the input path against the default file system
indicated by the configuration
This does not require a FileSystem handle in most cases - only requires the Filesystem URI.
|
static org.apache.hadoop.fs.Path |
makeRelative(org.apache.hadoop.fs.Path parentPath,
org.apache.hadoop.fs.Path childPath)
Returns a relative path wrt the parent path.
|
static boolean |
mkdir(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path f,
org.apache.hadoop.conf.Configuration conf)
Creates the directory and all necessary parent directories.
|
static boolean |
moveToTrash(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path f,
org.apache.hadoop.conf.Configuration conf,
boolean purge)
Move a particular file or directory to the trash.
|
static boolean |
pathsContainNoScheme(Collection<org.apache.hadoop.fs.Path> paths)
Return whenever all paths in the collection are schemaless
|
static void |
populateParentPaths(Set<org.apache.hadoop.fs.Path> parents,
org.apache.hadoop.fs.Path path) |
static void |
readFully(InputStream stream,
int length,
ByteBuffer bb)
Reads length bytes of data from the stream into the byte buffer.
|
static boolean |
rename(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path sourcePath,
org.apache.hadoop.fs.Path destPath,
org.apache.hadoop.conf.Configuration conf) |
static boolean |
shouldPreserveXAttrs(HiveConf conf,
org.apache.hadoop.fs.FileSystem srcFS,
org.apache.hadoop.fs.FileSystem dstFS) |
static String |
unescapePathName(String path) |
public static final int IO_ERROR_SLEEP_TIME
public static final org.apache.hadoop.fs.PathFilter HIDDEN_FILES_PATH_FILTER
public static final org.apache.hadoop.fs.PathFilter STAGING_DIR_PATH_FILTER
public static org.apache.hadoop.fs.Path makeQualified(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf) throws IOException
path
- path to be fully qualifiedconf
- Configuration fileIOException
public static String makePartName(List<String> partCols, List<String> vals, String defaultStr)
partCols
- The partition keys' namesvals
- The partition valuesdefaultStr
- The default name given to a partition value if the respective value is empty or null.public static String makeDefaultListBucketingDirName(List<String> skewedCols, String name)
skewedCols
- name
- public static String makeListBucketingDirName(List<String> lbCols, List<String> vals)
lbCols
- The skewed keys' namesvals
- The skewed valuespublic static String escapePathName(String path, String defaultPath)
path
- The path to escape.defaultPath
- The default name for the path, if the given path is empty or null.public static void listStatusRecursively(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus fileStatus, List<org.apache.hadoop.fs.FileStatus> results) throws IOException
fs
- file systemfileStatus
- starting point in file systemresults
- receives enumeration of all files foundIOException
public static org.apache.hadoop.fs.Path makeRelative(org.apache.hadoop.fs.Path parentPath, org.apache.hadoop.fs.Path childPath)
parentPath
- the parent path.childPath
- the child path.public static boolean isS3a(org.apache.hadoop.fs.FileSystem fs)
public static org.apache.hadoop.fs.FileStatus getPathOrParentThatExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
fs
- file systempath
- IOException
public static void checkFileAccessWithImpersonation(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus stat, org.apache.hadoop.fs.permission.FsAction action, String user) throws Exception
Exception
public static void checkFileAccessWithImpersonation(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus stat, org.apache.hadoop.fs.permission.FsAction action, String user, List<org.apache.hadoop.fs.FileStatus> children, org.apache.hadoop.fs.FileSystem fsAsUser) throws IOException, AccessControlException, InterruptedException, Exception
fs
- FileSystem of the path to checkstat
- FileStatus representing the fileaction
- FsAction that will be checkeduser
- User name of the user that will be checked for access. If the user name
is null or the same as the current user, no user impersonation will be done
and the check will be done as the current user. Otherwise the file access
check will be performed within a doAs() block to use the access privileges
of this user. In this case the user must be configured to impersonate other
users, otherwise this check will fail with error.children
- List of children to be collected. If this is null, no children are collected.
To be set only if this is a directoryIOException
AccessControlException
InterruptedException
Exception
public static org.apache.hadoop.security.UserGroupInformation getProxyUser(String user) throws LoginException, IOException
LoginException
IOException
public static void closeFs(org.apache.hadoop.security.UserGroupInformation proxyUser) throws IOException
IOException
public static org.apache.hadoop.fs.FileSystem getFsAsUser(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.security.UserGroupInformation proxyUser) throws IOException, InterruptedException
IOException
InterruptedException
public static boolean isActionPermittedForFileHierarchy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus fileStatus, String userName, org.apache.hadoop.fs.permission.FsAction action) throws Exception
fs
- fileStatus
- userName
- action
- IOException
Exception
public static boolean isActionPermittedForFileHierarchy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus fileStatus, String userName, org.apache.hadoop.fs.permission.FsAction action, boolean recurse) throws Exception
Exception
public static boolean isActionPermittedForFileHierarchy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus fileStatus, String userName, org.apache.hadoop.fs.permission.FsAction action, boolean recurse, org.apache.hadoop.fs.FileSystem fsAsUser) throws Exception
Exception
public static boolean isLocalFile(HiveConf conf, String fileName)
conf
- fileName
- public static boolean isLocalFile(HiveConf conf, URI fileUri)
conf
- fileUri
- public static boolean isOwnerOfFileHierarchy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus fileStatus, String userName) throws IOException, InterruptedException
IOException
InterruptedException
public static boolean isOwnerOfFileHierarchy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus fileStatus, String userName, boolean recurse) throws IOException, InterruptedException
IOException
InterruptedException
public static boolean checkIsOwnerOfFileHierarchy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus fileStatus, String userName, boolean recurse) throws IOException
IOException
public static boolean mkdir(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path f, org.apache.hadoop.conf.Configuration conf) throws IOException
fs
- FileSystem to usef
- path to create.conf
- Hive configurationIOException
- exception in creating the directorypublic static org.apache.hadoop.fs.Path makeAbsolute(org.apache.hadoop.fs.FileSystem fileSystem, org.apache.hadoop.fs.Path path) throws IOException
IOException
public static boolean copy(org.apache.hadoop.fs.FileSystem srcFS, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.FileSystem dstFS, org.apache.hadoop.fs.Path dst, boolean deleteSource, boolean overwrite, HiveConf conf) throws IOException
IOException
public static boolean copy(org.apache.hadoop.fs.FileSystem srcFS, org.apache.hadoop.fs.FileStatus srcStatus, org.apache.hadoop.fs.FileSystem dstFS, org.apache.hadoop.fs.Path dst, boolean deleteSource, boolean overwrite, boolean preserveXAttrs, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static boolean copy(org.apache.hadoop.fs.FileSystem srcFS, org.apache.hadoop.fs.Path[] srcs, org.apache.hadoop.fs.FileSystem dstFS, org.apache.hadoop.fs.Path dst, boolean deleteSource, boolean overwrite, boolean preserveXAttr, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static boolean shouldPreserveXAttrs(HiveConf conf, org.apache.hadoop.fs.FileSystem srcFS, org.apache.hadoop.fs.FileSystem dstFS) throws IOException
IOException
public static boolean distCp(org.apache.hadoop.fs.FileSystem srcFS, List<org.apache.hadoop.fs.Path> srcPaths, org.apache.hadoop.fs.Path dst, boolean deleteSource, org.apache.hadoop.security.UserGroupInformation proxyUser, HiveConf conf, HadoopShims shims) throws IOException
IOException
public static boolean distCpWithSnapshot(String oldSnapshot, String newSnapshot, List<org.apache.hadoop.fs.Path> srcPaths, org.apache.hadoop.fs.Path dst, boolean overwriteTarget, HiveConf conf, HadoopShims shims, org.apache.hadoop.security.UserGroupInformation proxyUser)
public static boolean moveToTrash(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path f, org.apache.hadoop.conf.Configuration conf, boolean purge) throws IOException
fs
- FileSystem to usef
- path of file or directory to move to trash.conf
- IOException
public static boolean rename(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path sourcePath, org.apache.hadoop.fs.Path destPath, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static boolean equalsFileSystem(org.apache.hadoop.fs.FileSystem fs1, org.apache.hadoop.fs.FileSystem fs2)
fs1
- fs2
- public static void checkDeletePermission(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf, String user) throws AccessControlException, InterruptedException, Exception
path
- conf
- user
- AccessControlException
InterruptedException
Exception
public static org.apache.hadoop.fs.FileStatus getFileStatusOrNull(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
fs
- file system to checkpath
- file system path to checkIOException
- if there is an I/O errorpublic static void deleteDirectory(File directory) throws IOException
IOException
public static File createTempFile(String lScratchDir, String prefix, String suffix) throws IOException
deleteTmpFile(File)
after the file is no longer required,
and has been deleted to avoid a memory leak.IOException
public static File createLocalDirsTempFile(String localDirList, String prefix, String suffix, boolean isDirectory) throws IOException
IOException
public static File createLocalDirsTempFile(org.apache.hadoop.conf.Configuration conf, String prefix, String suffix, boolean isDirectory) throws IOException
IOException
public static boolean deleteTmpFile(File tempFile)
public static boolean pathsContainNoScheme(Collection<org.apache.hadoop.fs.Path> paths)
paths
- public static org.apache.hadoop.fs.Path getParentRegardlessOfScheme(org.apache.hadoop.fs.Path path, Collection<org.apache.hadoop.fs.Path> candidates)
path
- candidates
- the candidate pathspublic static boolean isPathWithinSubtree(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.Path subtree)
path
- the path in questionsubtree
- public static void populateParentPaths(Set<org.apache.hadoop.fs.Path> parents, org.apache.hadoop.fs.Path path)
public static URI getURI(String path) throws URISyntaxException
URISyntaxException
public static Set<String> getJarFilesByPath(String pathString, org.apache.hadoop.conf.Configuration conf)
pathString
- the path string is the comma-separated path listpublic static void readFully(InputStream stream, int length, ByteBuffer bb) throws IOException
stream
- Stream to read from.length
- The number of bytes to read.bb
- The buffer to read into; the data is written at current position and then the
position is incremented by length.EOFException
- the length bytes cannot be read. The buffer position is not modified.IOException
public static int getSleepTime(int repeatNum)
repeatNum
- number of retry done so far.public static void deleteIfExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
fs
- FileSystempath
- Path to be deleted.public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> listStatusIterator(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.PathFilter filter) throws IOException
IOException
public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> listFiles(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, boolean recursive, org.apache.hadoop.fs.PathFilter filter) throws IOException
IOException
Copyright © 2022 The Apache Software Foundation. All rights reserved.