InputStream in = null; try { in = new URL("hdfs://host/path").openStream(); //操作输入流in,可以读取到文件的内容 } finally { IOUtils.closeStream(in); }
import java.io.InputStream; import java.net.URL; import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; import org.apache.hadoop.io.IOUtils; public class URLCat { static { URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); } public static void main(String[] args) throws Exception { InputStream in = null; try { in = new URL(args[0]).openStream(); IOUtils.copyBytes(in, System.out, 4096, false); } finally { IOUtils.closeStream(in); } } }
public static FileSystem get(Configuration conf) throws IOException; public static FileSystem get(URI uri, Configuration conf) throws IOException; public static FileSystem get(final URI uri, final Configuration conf, final String user) throws IOException, InterruptedException;
public static LocalFileSystem getLocal(Configuration conf) throws IOException;
public FSDataInputStream open(Path f) throws IOException; public abstract FSDataInputStream open(Path f, int bufferSize) throws IOException;
import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; public class FileSystemCat { public static void main(String[] args) throws Exception { String uri = args[0]; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); //System.out.println(fs.getClass().getName()); //这里可以看到得到的实例是DistributedFileSystem,因为core-site.xml里配的是hdfs FSDataInputStream in = null; try { in = fs.open(new Path(uri)); IOUtils.copyBytes(in, System.out, 4096, false); } finally { IOUtils.closeStream(in); } } }
in.seek(0); IOUtils.copyBytes(in, System.out, 4096, false);
public int read(long position, byte[] buffer, int offset, int length) throws IOException; public void readFully(long position, byte[] buffer, int offset, int length) throws IOException; public void readFully(long position, byte[] buffer) throws IOException;
public FSDataOutputStream create(Path f) throws IOException;
public FSDataOutputStream create(Path f, Progressable progress) throws IOException;
public interface Progressable { public void progress(); }
public FSDataOutputStream append(Path f) throws IOException;
import java.io.BufferedInputStream; import java.io.FileInputStream; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Progressable; public class FileCopyWithProgress { public static void main(String[] args) throws Exception { String localSrc = args[0]; String dst = args[1]; InputStream in = new BufferedInputStream(new FileInputStream(localSrc)); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dst), conf); OutputStream out = fs.create(new Path(dst), new Progressable() { @Override public void progress() { System.out.print("."); // try { // Thread.sleep(1000); // } catch (Exception e) { // e.printStackTrace(); // } } }); IOUtils.copyBytes(in, out, 4096, true); System.out.println(); System.out.println("end."); } }
public long getPos() throws IOException;
import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.*; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; import org.junit.Before; import org.junit.Test; public class ShowFileStatusTest { private static final String SYSPROP_KEY = "test.build.data"; /** MiniDFSCluster类在hadoop-hdfs-2.4.1-tests.jar中,是一个专门用于测试的in-process HDFS集群 */ private MiniDFSCluster cluster; private FileSystem fs; @Before public void setUp() throws IOException { Configuration conf = new Configuration(); String sysprop = System.getProperty(SYSPROP_KEY); if (sysprop == null) { System.setProperty(SYSPROP_KEY, "/tmp"); } cluster = new MiniDFSCluster(conf, 1, true, null); fs = cluster.getFileSystem(); OutputStream out = fs.create(new Path("/dir/file")); out.write("content".getBytes("UTF-8")); out.close(); } @After public void tearDown() throws IOException { if (fs != null) { fs.close(); } if (cluster != null) { cluster.shutdown(); } } @Test(expected = FileNotFoundException.class) public void throwsFileNotFoundForNonExistentFile() throws IOException { fs.getFileStatus(new Path("no-such-file")); } @Test public void fileStatusForFile() throws IOException { Path file = new Path("/dir/file"); FileStatus stat = fs.getFileStatus(file); assertThat(stat.getPath().toUri().getPath(), is("/dir/file")); assertThat(stat.isDirectory(), is(false)); assertThat(stat.getLen(), is(7L)); assertTrue(stat.getModificationTime() <= System.currentTimeMillis()); assertThat(stat.getReplication(), is((short)1)); assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L)); assertThat(stat.getOwner(), is("norris")); assertThat(stat.getGroup(), is("supergroup")); assertThat(stat.getPermission().toString(), is("rw-r--r--")); } @Test public void fileStatusForDirectory() throws IOException { Path dir = new Path("/dir"); FileStatus stat = fs.getFileStatus(dir); assertThat(stat.getPath().toUri().getPath(), is("/dir")); assertThat(stat.isDirectory(), is(true)); assertThat(stat.getLen(), is(0L)); assertTrue(stat.getModificationTime() <= System.currentTimeMillis()); assertThat(stat.getReplication(), is((short)0)); assertThat(stat.getBlockSize(), is(0L)); assertThat(stat.getOwner(), is("norris")); assertThat(stat.getGroup(), is("supergroup")); assertThat(stat.getPermission().toString(), is("rwxr-xr-x")); } }
原文:http://blog.csdn.net/norriszhang/article/details/39648857