| Method from org.apache.hadoop.fs.FileSystem Detail: |
|---|
 protected void checkPath(Path path) {
URI uri = path.toUri();
if (uri.getScheme() == null) // fs is relative
return;
String thisAuthority = this.getUri().getAuthority();
String thatAuthority = uri.getAuthority();
if (!(this.getUri().getScheme().equals(uri.getScheme()) &&
(thisAuthority == thatAuthority ||
(thisAuthority != null && thisAuthority.equals(thatAuthority))))){
throw new IllegalArgumentException("Wrong FS: "+path+
", expected: "+this.getUri());
}
}
Check that a Path belongs to this FileSystem. |
public void close() throws IOException {
URI uri = getUri();
synchronized (FileSystem.class) {
Map< String,FileSystem > authorityToFs = CACHE.get(uri.getScheme());
if (authorityToFs != null) {
authorityToFs.remove(uri.getAuthority());
}
}
} No more filesystem operations are needed. Will
release any held locks. |
public static synchronized void closeAll() throws IOException {
for(Map< String, FileSystem > fss : CACHE.values()){
for(FileSystem fs : fss.values()){
fs.close();
}
}
} Close all cached filesystems. Be sure those filesystems are not
used anymore. |
public void completeLocalOutput(Path fsOutputFile,
Path tmpLocalFile) throws IOException {
moveFromLocalFile(tmpLocalFile, fsOutputFile);
} Called when we're all done writing to the target. A local FS will
do nothing, because we've written to exactly the right place. A remote
FS will copy the contents of tmpLocalFile to the correct target at
fsOutputFile. |
public void copyFromLocalFile(Path src,
Path dst) throws IOException {
copyFromLocalFile(false, src, dst);
} The src file is on the local disk. Add it to FS at
the given dst name and the source is kept intact afterwards |
public void copyFromLocalFile(boolean delSrc,
Path src,
Path dst) throws IOException {
FileUtil.copy(getLocal(getConf()), src, this, dst, delSrc, getConf());
} The src file is on the local disk. Add it to FS at
the given dst name.
delSrc indicates if the source should be removed |
public void copyToLocalFile(Path src,
Path dst) throws IOException {
copyToLocalFile(false, src, dst);
} The src file is under FS, and the dst is on the local disk.
Copy it from FS control to the local dst name. |
public void copyToLocalFile(boolean delSrc,
Path src,
Path dst) throws IOException {
FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc, getConf());
} The src file is under FS, and the dst is on the local disk.
Copy it from FS control to the local dst name.
delSrc indicates if the src will be removed or not. |
public FSDataOutputStream create(Path f) throws IOException {
return create(f, true,
getConf().getInt("io.file.buffer.size", 4096),
getDefaultReplication(),
getDefaultBlockSize());
} Opens an FSDataOutputStream at the indicated Path.
Files are overwritten by default. |
public FSDataOutputStream create(Path f,
Progressable progress) throws IOException {
return create(f, true,
getConf().getInt("io.file.buffer.size", 4096),
getDefaultReplication(),
getDefaultBlockSize(), progress);
} Create an FSDataOutputStream at the indicated Path with write-progress
reporting.
Files are overwritten by default. |
public FSDataOutputStream create(Path f,
short replication) throws IOException {
return create(f, true,
getConf().getInt("io.file.buffer.size", 4096),
replication,
getDefaultBlockSize());
} Opens an FSDataOutputStream at the indicated Path.
Files are overwritten by default. |
public FSDataOutputStream create(Path f,
short replication,
Progressable progress) throws IOException {
return create(f, true,
getConf().getInt("io.file.buffer.size", 4096),
replication,
getDefaultBlockSize(), progress);
} Opens an FSDataOutputStream at the indicated Path with write-progress
reporting.
Files are overwritten by default. |
public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize) throws IOException {
return create(f, overwrite, bufferSize,
getDefaultReplication(),
getDefaultBlockSize());
} Opens an FSDataOutputStream at the indicated Path. |
public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
Progressable progress) throws IOException {
return create(f, overwrite, bufferSize,
getDefaultReplication(),
getDefaultBlockSize(), progress);
} Opens an FSDataOutputStream at the indicated Path with write-progress
reporting. |
public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
short replication,
long blockSize) throws IOException {
return create(f, overwrite, bufferSize, replication, blockSize, null);
} Opens an FSDataOutputStream at the indicated Path. |
 abstract public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress) throws IOException
Opens an FSDataOutputStream at the indicated Path with write-progress
reporting. |
public boolean createNewFile(Path f) throws IOException {
if (exists(f)) {
return false;
} else {
create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close();
return true;
}
} Creates the given Path as a brand-new zero-length file. If
create fails, or if it already existed, return false. |
abstract public boolean delete(Path f) throws IOException |
abstract public boolean exists(Path f) throws IOException |
public static FileSystem get(Configuration conf) throws IOException {
return getNamed(conf.get("fs.default.name", "file:///"), conf);
} Returns the configured filesystem implementation. |
public static synchronized FileSystem get(URI uri,
Configuration conf) throws IOException {
String scheme = uri.getScheme();
String authority = uri.getAuthority();
if (scheme == null) { // no scheme: use default FS
return get(conf);
}
Map< String,FileSystem > authorityToFs = CACHE.get(scheme);
if (authorityToFs == null) {
authorityToFs = new HashMap< String,FileSystem >();
CACHE.put(scheme, authorityToFs);
}
FileSystem fs = authorityToFs.get(authority);
if (fs == null) {
Class fsClass = conf.getClass("fs."+scheme+".impl", null);
if (fsClass == null) {
throw new IOException("No FileSystem for scheme: " + scheme);
}
fs = (FileSystem)ReflectionUtils.newInstance(fsClass, conf);
fs.initialize(uri, conf);
authorityToFs.put(authority, fs);
}
return fs;
} Returns the FileSystem for this URI's scheme and authority. The scheme
of the URI determines a configuration property name,
fs.scheme.class whose value names the FileSystem class.
The entire URI is passed to the FileSystem instance's initialize method. |
public long getBlockSize(Path f) throws IOException {
return getFileStatus(f).getBlockSize();
} Deprecated! Use - getFileStatus() instead
|
public long getContentLength(Path f) throws IOException {
if (!isDirectory(f)) {
// f is a file
return getLength(f);
}
// f is a diretory
Path[] contents = listPaths(f);
long size = 0;
for(int i=0; i< contents.length; i++) {
size += getContentLength(contents[i]);
}
return size;
} Return the number of bytes of the given path
If f is a file, return the size of the file;
If f is a directory, return the size of the directory tree |
public long getDefaultBlockSize() {
// default to 32MB: large enough to minimize the impact of seeks
return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
} Return the number of bytes that large input files should be optimally
be split into to minimize i/o time. |
public short getDefaultReplication() {
return 1;
} Get the default replication. |
public String[][] getFileCacheHints(Path f,
long start,
long len) throws IOException {
if (!exists(f)) {
return null;
} else {
String result[][] = new String[1][];
result[0] = new String[1];
result[0][0] = "localhost";
return result;
}
} Return a 2D array of size 1x1 or greater, containing hostnames
where portions of the given file can be found. For a nonexistent
file or regions, null will be returned.
This call is most helpful with DFS, where it returns
hostnames of machines that contain the given file.
The FileSystem will simply return an elt containing 'localhost'. |
abstract public FileStatus getFileStatus(Path f) throws IOException |
public long getLength(Path f) throws IOException {
return getFileStatus(f).getLen();
} Deprecated! Use - getFileStatus() instead
|
public static LocalFileSystem getLocal(Configuration conf) throws IOException {
return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
} |
public String getName() {
return getUri().toString();
} Deprecated! call - #getUri() instead.
|
public static FileSystem getNamed(String name,
Configuration conf) throws IOException {
// convert old-format name to new-format name
if (name.equals("local")) { // "local" is now "file:///".
name = "file:///";
} else if (name.indexOf('/")==-1) { // unqualified is "hdfs://"
name = "hdfs://"+name;
}
return get(URI.create(name), conf);
} Deprecated! call - #get(URI,Configuration) instead.
|
public short getReplication(Path src) throws IOException {
return getFileStatus(src).getReplication();
} Deprecated! Use - getFileStatus() instead
|
abstract public URI getUri() Returns a URI whose scheme and authority identify this FileSystem. |
public long getUsed() throws IOException {
long used = 0;
Path[] files = listPaths(new Path("/"));
for(Path file:files){
used += getContentLength(file);
}
return used;
} Return the total size of all files in the filesystem. |
abstract public Path getWorkingDirectory() Get the current working directory for the given file system |
public Path[] globPaths(Path filePattern) throws IOException {
return globPaths(filePattern, DEFAULT_FILTER);
} Return all the files that match filePattern and are not checksum
files. Results are sorted by their names.
A filename pattern is composed of regular characters and
special pattern matching characters, which are:
-
- ?
- Matches any single character.
- *
- Matches zero or more characters.
- [abc]
- Matches a single character from character set
{a,b,c}.
- [a-b]
- Matches a single character from the character range
{a...b}. Note that character a must be
lexicographically less than or equal to character b.
- [^a]
- Matches a single character that is not from character set or range
{a}. Note that the ^ character must occur
immediately to the right of the opening bracket.
- \c
- Removes (escapes) any special meaning of character c.
|
public Path[] globPaths(Path filePattern,
PathFilter filter) throws IOException {
Path [] parents = new Path[1];
int level = 0;
String filename = filePattern.toUri().getPath();
if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
parents[0] = filePattern;
return parents;
}
String [] components = filename.split(Path.SEPARATOR);
if (filePattern.isAbsolute()) {
parents[0] = new Path(Path.SEPARATOR);
level = 1;
} else {
parents[0] = new Path(Path.CUR_DIR);
}
Path[] results = globPathsLevel(parents, components, level, filter);
Arrays.sort(results);
return results;
} glob all the file names that matches filePattern
and is accepted by filter. |
abstract public void initialize(URI name,
Configuration conf) throws IOException Called after a new FileSystem instance is constructed. |
public boolean isDirectory(Path f) throws IOException {
try {
return getFileStatus(f).isDir();
} catch (IOException e) {
return false; // f does not exist
}
} Deprecated! Use - getFileStatus() instead
|
public boolean isFile(Path f) throws IOException {
if (exists(f) && !isDirectory(f)) {
return true;
} else {
return false;
}
} True iff the named path is a regular file. |
abstract public Path[] listPaths(Path f) throws IOException List files in a directory. |
public Path[] listPaths(Path[] files) throws IOException {
return listPaths(files, DEFAULT_FILTER);
} Filter files in the given pathes using the default checksum filter. |
public Path[] listPaths(Path f,
PathFilter filter) throws IOException {
ArrayList< Path > results = new ArrayList< Path >();
listPaths(results, f, filter);
return (Path[]) results.toArray(new Path[results.size()]);
} Filter files in a directory. |
public Path[] listPaths(Path[] files,
PathFilter filter) throws IOException {
ArrayList< Path > results = new ArrayList< Path >();
for(int i=0; i< files.length; i++) {
listPaths(results, files[i], filter);
}
return (Path[]) results.toArray(new Path[results.size()]);
} Filter files in a list directories using user-supplied path filter. |
public void lock(Path f,
boolean shared) throws IOException {
} Deprecated! FS - does not support file locks anymore.
Obtain a lock on the given Path |
public Path makeQualified(Path path) {
checkPath(path);
return path.makeQualified(this);
} Make sure that a path specifies a FileSystem. |
abstract public boolean mkdirs(Path f) throws IOException Make the given file and all non-existent parents into
directories. Has the semantics of Unix 'mkdir -p'.
Existence of the directory hierarchy is not an error. |
public void moveFromLocalFile(Path src,
Path dst) throws IOException {
copyFromLocalFile(true, src, dst);
} The src file is on the local disk. Add it to FS at
the given dst name, removing the source afterwards. |
public void moveToLocalFile(Path src,
Path dst) throws IOException {
copyToLocalFile(true, src, dst);
} The src file is under FS, and the dst is on the local disk.
Copy it from FS control to the local dst name.
Remove the source afterwards |
public FSDataInputStream open(Path f) throws IOException {
return open(f, getConf().getInt("io.file.buffer.size", 4096));
} Opens an FSDataInputStream at the indicated Path. |
abstract public FSDataInputStream open(Path f,
int bufferSize) throws IOException Opens an FSDataInputStream at the indicated Path. |
public static FileSystem parseArgs(String[] argv,
int i,
Configuration conf) throws IOException {
/**
if (argv.length - i < 1) {
throw new IOException("Must indicate filesystem type for DFS");
}
*/
int orig = i;
FileSystem fs = null;
String cmd = argv[i];
if ("-dfs".equals(cmd)) {
i++;
InetSocketAddress addr = DataNode.createSocketAddr(argv[i++]);
fs = new DistributedFileSystem(addr, conf);
} else if ("-local".equals(cmd)) {
i++;
fs = FileSystem.getLocal(conf);
} else {
fs = get(conf); // using default
LOG.info("No FS indicated, using default:"+fs.getName());
}
System.arraycopy(argv, i, argv, orig, argv.length - i);
for (int j = argv.length - i; j < argv.length; j++) {
argv[j] = null;
}
return fs;
} Parse the cmd-line args, starting at i. Remove consumed args
from array. We expect param in the form:
'-local | -dfs ' |
public void release(Path f) throws IOException {
} Deprecated! FS - does not support file locks anymore.
|
abstract public boolean rename(Path src,
Path dst) throws IOException Renames Path src to Path dst. Can take place on local fs
or remote DFS. |
public boolean setReplication(Path src,
short replication) throws IOException {
return true;
} Set replication for an existing file. |
abstract public void setWorkingDirectory(Path new_dir) Set the current working directory for the given file system. All relative
paths will be resolved relative to it. |
public Path startLocalOutput(Path fsOutputFile,
Path tmpLocalFile) throws IOException {
return tmpLocalFile;
} Returns a local File that the user can write output to. The caller
provides both the eventual FS target name and the local working
file. If the FS is local, we write directly into the target. If
the FS is remote, we write into the tmp local area. |
org.apache.hadoop.conf.Configured
Save This Page