@InterfaceAudience.Public @InterfaceStability.Evolving public class AdlFileSystem extends FileSystem
Modifier and Type | Field and Description |
---|---|
static String |
SCHEME |
DEFAULT_FS, FS_DEFAULT_NAME_KEY, SHUTDOWN_HOOK_PRIORITY, statistics, TRASH_PREFIX, USER_HOME_PREFIX
Constructor and Description |
---|
AdlFileSystem() |
Modifier and Type | Method and Description |
---|---|
void |
access(Path path,
FsAction mode)
Checks if the user can access a path.
|
FSDataOutputStream |
append(Path f,
int bufferSize,
Progressable progress)
Append to an existing file (optional operation).
|
void |
concat(Path trg,
Path[] srcs)
Concat existing files together.
|
FSDataOutputStream |
create(Path f,
FsPermission permission,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress)
Create call semantic is handled differently in case of ADL.
|
FSDataOutputStream |
createNonRecursive(Path f,
FsPermission permission,
EnumSet<CreateFlag> flags,
int bufferSize,
short replication,
long blockSize,
Progressable progress)
Deprecated.
API only for 0.20-append
|
boolean |
delete(Path path,
boolean recursive)
Delete a file.
|
static String |
getAccountNameFromFQDN(String accountFQDN)
Gets ADL account name from ADL FQDN.
|
AclStatus |
getAclStatus(Path path)
Gets the ACL of a file or directory.
|
com.microsoft.azure.datalake.store.ADLStoreClient |
getAdlClient() |
long |
getBlockSize(Path f)
Deprecated.
Use getFileStatus() instead
|
ContentSummary |
getContentSummary(Path f)
Return the
ContentSummary of a given Path . |
protected AzureADTokenProvider |
getCustomAccessTokenProvider(Configuration conf)
This method is provided for convenience for derived classes to define
custom
AzureADTokenProvider instance. |
long |
getDefaultBlockSize()
Deprecated.
use
getDefaultBlockSize(Path) instead |
long |
getDefaultBlockSize(Path f)
Return the number of bytes that large input files should be optimally
be split into to minimize i/o time.
|
int |
getDefaultPort()
Get the default port for this FileSystem.
|
FileStatus |
getFileStatus(Path f)
Return a file status object that represents the path.
|
Path |
getHomeDirectory()
Constructing home directory locally is fine as long as Hadoop
local user name and ADL user name relationship story is not fully baked
yet.
|
short |
getReplication(Path src)
Deprecated.
Use getFileStatus() instead
|
String |
getScheme()
Return the protocol scheme for this FileSystem.
|
protected String |
getTransportScheme() |
URI |
getUri()
Returns a URI which identifies this FileSystem.
|
Path |
getWorkingDirectory()
Get the current working directory for the given file system.
|
void |
initialize(URI storeUri,
Configuration originalConf)
Called after a new FileSystem instance is constructed.
|
FileStatus[] |
listStatus(Path f)
List the statuses of the files/directories in the given path if the path is
a directory.
|
boolean |
mkdirs(Path path,
FsPermission permission)
Make the given file and all non-existent parents into
directories.
|
void |
modifyAclEntries(Path path,
List<AclEntry> aclSpec)
Modifies ACL entries of files and directories.
|
FSDataInputStream |
open(Path f,
int buffersize)
Open call semantic is handled differently in case of ADL.
|
static Configuration |
propagateAccountOptions(Configuration source,
String accountName)
Propagates account-specific settings into generic ADL configuration keys.
|
void |
removeAcl(Path path)
Removes all but the base ACL entries of files and directories.
|
void |
removeAclEntries(Path path,
List<AclEntry> aclSpec)
Removes ACL entries from files and directories.
|
void |
removeDefaultAcl(Path path)
Removes all default ACL entries from files and directories.
|
boolean |
rename(Path src,
Path dst)
Renames Path src to Path dst.
|
void |
rename(Path src,
Path dst,
org.apache.hadoop.fs.Options.Rename... options)
Deprecated.
|
void |
setAcl(Path path,
List<AclEntry> aclSpec)
Fully replaces ACL of files and directories, discarding all existing
entries.
|
void |
setOwner(Path path,
String owner,
String group)
Set owner of a path (i.e.
|
void |
setPermission(Path path,
FsPermission permission)
Set permission of a path.
|
boolean |
setReplication(Path p,
short replication)
Azure data lake does not support user configuration for data replication
hence not leaving system to query on
azure data lake.
|
void |
setUserGroupRepresentationAsUPN(boolean enableUPN) |
void |
setWorkingDirectory(Path dir)
Set the current working directory for the given file system.
|
boolean |
supportsSymlinks()
|
append, append, appendFile, areSymlinksEnabled, cancelDeleteOnExit, canonicalizeUri, checkPath, clearStatistics, close, closeAll, closeAllForUGI, completeLocalOutput, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyToLocalFile, copyToLocalFile, copyToLocalFile, create, create, create, create, create, create, create, create, create, create, create, create, createFile, createNewFile, createNonRecursive, createNonRecursive, createSnapshot, createSnapshot, createSymlink, delete, deleteOnExit, deleteSnapshot, enableSymlinks, exists, fixRelativePart, get, get, get, getAllStatistics, getAllStoragePolicies, getCanonicalServiceName, getCanonicalUri, getDefaultReplication, getDefaultReplication, getDefaultUri, getFileBlockLocations, getFileBlockLocations, getFileChecksum, getFileChecksum, getFileLinkStatus, getFileSystemClass, getFSofPath, getGlobalStorageStatistics, getInitialWorkingDirectory, getLength, getLinkTarget, getLocal, getName, getNamed, getQuotaUsage, getServerDefaults, getServerDefaults, getStatistics, getStatistics, getStatus, getStatus, getStoragePolicy, getStorageStatistics, getTrashRoot, getTrashRoots, getUsed, getUsed, getXAttr, getXAttrs, getXAttrs, globStatus, globStatus, isDirectory, isFile, listCorruptFileBlocks, listFiles, listLocatedStatus, listLocatedStatus, listStatus, listStatus, listStatus, listStatusIterator, listXAttrs, makeQualified, mkdirs, mkdirs, moveFromLocalFile, moveFromLocalFile, moveToLocalFile, newInstance, newInstance, newInstance, newInstanceLocal, open, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, removeXAttr, renameSnapshot, resolveLink, resolvePath, setDefaultUri, setDefaultUri, setStoragePolicy, setTimes, setVerifyChecksum, setWriteChecksum, setXAttr, setXAttr, startLocalOutput, truncate, unsetStoragePolicy
getConf, setConf
public static final String SCHEME
public String getScheme()
FileSystem
This implementation throws an UnsupportedOperationException
.
getScheme
in class FileSystem
public URI getUri()
FileSystem
getUri
in class FileSystem
public int getDefaultPort()
FileSystem
getDefaultPort
in class FileSystem
public boolean supportsSymlinks()
FileSystem
supportsSymlinks
in class FileSystem
public void initialize(URI storeUri, Configuration originalConf) throws IOException
initialize
in class FileSystem
storeUri
- a uri whose authority section names the host, port,
etc. for this FileSystemoriginalConf
- the configuration to use for the FS. The account-
specific options are patched over the base ones
before any use is made of the config.IOException
- on any failure to initialize this instance.protected AzureADTokenProvider getCustomAccessTokenProvider(Configuration conf) throws IOException
AzureADTokenProvider
instance.
In order to ensure secure hadoop infrastructure and user context for which
respective AdlFileSystem
instance is initialized,
Loading AzureADTokenProvider
is not sufficient.
The order of loading AzureADTokenProvider
is to first invoke
getCustomAccessTokenProvider(Configuration)
, If method return null
which means no implementation provided by derived classes, then
configuration object is loaded to retrieve token configuration as specified
is documentation.
Custom token management takes the higher precedence during initialization.conf
- Configuration objectAzureADTokenProvider
token management
is specified.IOException
- if failed to initialize token provider.public com.microsoft.azure.datalake.store.ADLStoreClient getAdlClient()
public Path getHomeDirectory()
getHomeDirectory
in class FileSystem
public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
create
in class FileSystem
f
- File pathpermission
- Access permission for the newly created fileoverwrite
- Remove existing file and recreate new one if true
otherwise throw error if file existbufferSize
- Buffer size, ADL backend does not honourreplication
- Replication count, ADL backend does not honourblockSize
- Block size, ADL backend does not honourprogress
- Progress indicatorIOException
- when system error, internal server error or user errorFileSystem.setPermission(Path, FsPermission)
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
createNonRecursive
in class FileSystem
f
- the file name to openpermission
- Access permission for the newly created fileflags
- CreateFlag
s to use for this stream.bufferSize
- the size of the buffer to be used. ADL backend does
not honourreplication
- required block replication for the file. ADL backend
does not honourblockSize
- Block size, ADL backend does not honourprogress
- Progress indicatorIOException
- when system error, internal server error or user errorsetPermission(Path, FsPermission)
public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException
append
in class FileSystem
f
- the existing file to be appended.bufferSize
- the size of the buffer to be used. ADL backend does
not honourprogress
- Progress indicatorIOException
- when system error, internal server error or user errorpublic boolean setReplication(Path p, short replication) throws IOException
setReplication
in class FileSystem
p
- Not honouredreplication
- Not honouredIOException
- No exception would not thrown in this case however
aligning with parent api definition.public FSDataInputStream open(Path f, int buffersize) throws IOException
open
in class FileSystem
f
- File pathbuffersize
- Buffer size, Not honouredIOException
- when system error, internal server error or user errorpublic FileStatus getFileStatus(Path f) throws IOException
getFileStatus
in class FileSystem
f
- The path we want information fromIOException
- when the path does not exist or any other error;
IOException see specific implementationpublic FileStatus[] listStatus(Path f) throws IOException
listStatus
in class FileSystem
f
- given pathIOException
- when the path does not exist or any other error;
IOException see specific implementationpublic boolean rename(Path src, Path dst) throws IOException
rename
in class FileSystem
src
- path to be renameddst
- new path after renameIOException
- on failure@Deprecated public void rename(Path src, Path dst, org.apache.hadoop.fs.Options.Rename... options) throws IOException
FileSystem
If OVERWRITE option is not passed as an argument, rename fails if the dst already exists.
If OVERWRITE option is passed as an argument, rename overwrites the dst if it is a file or an empty directory. Rename fails if dst is a non-empty directory.
Note that atomicity of rename is dependent on the file system implementation. Please refer to the file system documentation for details. This default implementation is non atomic.
This method is deprecated since it is a temporary method added to support the transition from FileSystem to FileContext for user applications.
rename
in class FileSystem
src
- path to be renameddst
- new path after renameFileNotFoundException
- src path does not exist, or the parent
path of dst does not exist.FileAlreadyExistsException
- dest path exists and is a fileParentNotDirectoryException
- if the parent path of dest is not
a directoryIOException
- on failurepublic void concat(Path trg, Path[] srcs) throws IOException
concat
in class FileSystem
trg
- the path to the target destination.srcs
- the paths to the sources to use for the concatenation.IOException
- when system error, internal server error or user errorpublic boolean delete(Path path, boolean recursive) throws IOException
delete
in class FileSystem
path
- the path to delete.recursive
- if path is a directory and set to
true, the directory is deleted else throws an exception.
In case of a file the recursive can be set to either
true or false.IOException
- when system error, internal server error or user errorpublic boolean mkdirs(Path path, FsPermission permission) throws IOException
mkdirs
in class FileSystem
path
- path to createpermission
- to apply to pathIOException
- IO failurepublic void setOwner(Path path, String owner, String group) throws IOException
setOwner
in class FileSystem
path
- The pathowner
- If it is null, the original username remains unchanged.group
- If it is null, the original groupname remains unchanged.IOException
- IO failurepublic void setPermission(Path path, FsPermission permission) throws IOException
setPermission
in class FileSystem
path
- The pathpermission
- Access permissionIOException
- IO failurepublic void modifyAclEntries(Path path, List<AclEntry> aclSpec) throws IOException
modifyAclEntries
in class FileSystem
path
- Path to modifyaclSpec
- List of AclEntry describing modificationsIOException
- if an ACL could not be modifiedpublic void removeAclEntries(Path path, List<AclEntry> aclSpec) throws IOException
removeAclEntries
in class FileSystem
path
- Path to modifyaclSpec
- List of AclEntry describing entries to removeIOException
- if an ACL could not be modifiedpublic void removeDefaultAcl(Path path) throws IOException
removeDefaultAcl
in class FileSystem
path
- Path to modifyIOException
- if an ACL could not be modifiedpublic void removeAcl(Path path) throws IOException
removeAcl
in class FileSystem
path
- Path to modifyIOException
- if an ACL could not be removedpublic void setAcl(Path path, List<AclEntry> aclSpec) throws IOException
setAcl
in class FileSystem
path
- Path to modifyaclSpec
- List of AclEntry describing modifications, must include
entries for user, group, and others for compatibility with
permission bits.IOException
- if an ACL could not be modifiedpublic AclStatus getAclStatus(Path path) throws IOException
getAclStatus
in class FileSystem
path
- Path to getIOException
- if an ACL could not be readpublic void access(Path path, FsAction mode) throws IOException
AccessControlException
.path
- Path to checkmode
- type of access to checkAccessControlException
- if access is deniedFileNotFoundException
- if the path does not existIOException
- see specific implementationpublic ContentSummary getContentSummary(Path f) throws IOException
ContentSummary
of a given Path
.getContentSummary
in class FileSystem
f
- path to useFileNotFoundException
- if the path does not resolveIOException
- IO failureprotected String getTransportScheme()
public Path getWorkingDirectory()
getWorkingDirectory
in class FileSystem
public void setWorkingDirectory(Path dir)
setWorkingDirectory
in class FileSystem
dir
- Working directory path.@Deprecated public long getDefaultBlockSize()
getDefaultBlockSize(Path)
insteadgetDefaultBlockSize
in class FileSystem
public long getDefaultBlockSize(Path f)
getDefaultBlockSize
in class FileSystem
f
- path of file@Deprecated public long getBlockSize(Path f) throws IOException
FileSystem
getBlockSize
in class FileSystem
f
- the filenameFileNotFoundException
- if the path is not presentIOException
- IO failure@Deprecated public short getReplication(Path src)
getReplication
in class FileSystem
src
- file namepublic void setUserGroupRepresentationAsUPN(boolean enableUPN)
public static String getAccountNameFromFQDN(String accountFQDN)
accountFQDN
- ADL account fqdnpublic static Configuration propagateAccountOptions(Configuration source, String accountName)
fs.adl.account.${account_name}.key
to
fs.adl.key
, for all values of "key"
The source of the updated property is set to the key name of the account
property, to aid in diagnostics of where things came from.
Returns a new configuration. Why the clone?
You can use the same conf for different filesystems, and the original
values are not updated.source
- Source Configuration objectaccountName
- account name. Must not be emptyCopyright © 2019 Apache Software Foundation. All rights reserved.