Abstract Checksumed FileSystem.
It provide a basice implementation of a Checksumed FileSystem,
which creates a checksum file for each raw file.
It generates & verifies checksums at the client side.
| Method from org.apache.hadoop.fs.ChecksumFileSystem Detail: |
public void completeLocalOutput(Path fsOutputFile,
Path tmpLocalFile) throws IOException {
moveFromLocalFile(tmpLocalFile, fsOutputFile);
}
|
public void copyFromLocalFile(boolean delSrc,
Path src,
Path dst) throws IOException {
Configuration conf = getConf();
FileUtil.copy(getLocal(conf), src, this, dst, delSrc, conf);
}
|
public void copyToLocalFile(boolean delSrc,
Path src,
Path dst) throws IOException {
Configuration conf = getConf();
FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf);
}
The src file is under FS, and the dst is on the local disk.
Copy it from FS control to the local dst name. |
public void copyToLocalFile(Path src,
Path dst,
boolean copyCrc) throws IOException {
if (!fs.isDirectory(src)) { // source is a file
fs.copyToLocalFile(src, dst);
FileSystem localFs = getLocal(getConf());
if (localFs instanceof ChecksumFileSystem) {
localFs = ((ChecksumFileSystem) localFs).getRawFileSystem();
}
if (localFs.isDirectory(dst)) {
dst = new Path(dst, src.getName());
}
dst = getChecksumFile(dst);
if (localFs.exists(dst)) { //remove old local checksum file
localFs.delete(dst);
}
Path checksumFile = getChecksumFile(src);
if (copyCrc && fs.exists(checksumFile)) { //copy checksum file
fs.copyToLocalFile(checksumFile, dst);
}
} else {
Path[] srcs = listPaths(src);
for (Path srcFile : srcs) {
copyToLocalFile(srcFile, new Path(dst, srcFile.getName()), copyCrc);
}
}
}
The src file is under FS, and the dst is on the local disk.
Copy it from FS control to the local dst name.
If src and dst are directories, the copyCrc parameter
determines whether to copy CRC files. |
public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress) throws IOException {
Path parent = f.getParent();
if (parent != null && !mkdirs(parent)) {
throw new IOException("Mkdirs failed to create " + parent);
}
return new FSDataOutputStream(new ChecksumFSOutputSummer(
this, f, overwrite, bufferSize, replication, blockSize, progress));
}
Opens an FSDataOutputStream at the indicated Path with write-progress
reporting. |
public boolean delete(Path f) throws IOException {
if (fs.isDirectory(f)) {
return fs.delete(f);
} else {
Path checkFile = getChecksumFile(f);
if (fs.exists(checkFile)) {
fs.delete(checkFile);
}
return fs.delete(f);
}
}
Get rid of Path f, whether a true file or dir. |
public static double getApproxChkSumLength(long size) {
return ChecksumFSOutputSummer.CHKSUM_AS_FRACTION * size;
}
|
public int getBytesPerSum() {
return getConf().getInt("io.bytes.per.checksum", 512);
}
Return the bytes Per Checksum |
public Path getChecksumFile(Path file) {
return new Path(file.getParent(), "." + file.getName() + ".crc");
}
Return the name of the checksum file associated with a file. |
public long getChecksumFileLength(Path file,
long fileSize) {
return ChecksumFSOutputSummer.getChecksumLength(fileSize, getBytesPerSum());
}
Return the length of the checksum file given the size of the
actual file. |
public FileSystem getRawFileSystem() {
return fs;
}
|
public static boolean isChecksumFile(Path file) {
String name = file.getName();
return name.startsWith(".") && name.endsWith(".crc");
}
Return true iff file is a checksum file name. |
public Path[] listPaths(Path[] files) throws IOException {
return fs.listPaths(files, DEFAULT_FILTER);
}
Filter raw files in the given pathes using the default checksum filter. |
public Path[] listPaths(Path f) throws IOException {
return fs.listPaths(f, DEFAULT_FILTER);
}
Filter raw files in the given path using the default checksum filter. |
public void lock(Path f,
boolean shared) throws IOException {
if (fs.isDirectory(f)) {
fs.lock(f, shared);
} else {
Path checkFile = getChecksumFile(f);
if (fs.exists(checkFile)) {
fs.lock(checkFile, shared);
}
fs.lock(f, shared);
}
}
|
public boolean mkdirs(Path f) throws IOException {
return fs.mkdirs(f);
}
|
public FSDataInputStream open(Path f,
int bufferSize) throws IOException {
return new FSDataInputStream(
new ChecksumFSInputChecker(this, f, bufferSize) );
}
Opens an FSDataInputStream at the indicated Path. |
public void release(Path f) throws IOException {
if (fs.isDirectory(f)) {
fs.release(f);
} else {
Path checkFile = getChecksumFile(f);
if (fs.exists(checkFile)) {
fs.release(getChecksumFile(f));
}
fs.release(f);
}
}
|
public boolean rename(Path src,
Path dst) throws IOException {
if (fs.isDirectory(src)) {
return fs.rename(src, dst);
} else {
boolean value = fs.rename(src, dst);
if (!value)
return false;
Path checkFile = getChecksumFile(src);
if (fs.exists(checkFile)) { //try to rename checksum
if (fs.isDirectory(dst)) {
value = fs.rename(checkFile, dst);
} else {
value = fs.rename(checkFile, getChecksumFile(dst));
}
}
return value;
}
}
|
public boolean reportChecksumFailure(Path f,
FSDataInputStream in,
long inPos,
FSDataInputStream sums,
long sumsPos) {
return false;
}
Report a checksum error to the file system. |
public boolean setReplication(Path src,
short replication) throws IOException {
boolean value = fs.setReplication(src, replication);
if (!value)
return false;
Path checkFile = getChecksumFile(src);
if (exists(checkFile))
fs.setReplication(checkFile, replication);
return true;
}
Set replication for an existing file.
Implement the abstract setReplication of FileSystem |
public Path startLocalOutput(Path fsOutputFile,
Path tmpLocalFile) throws IOException {
return tmpLocalFile;
}
|