博客
关于我
强烈建议你试试无所不能的chatGPT,快点击我
HDFS文件目录操作代码
阅读量:5316 次
发布时间:2019-06-14

本文共 11018 字,大约阅读时间需要 36 分钟。

分布式文件系统HDFS中对文件/目录的相关操作代码,整理了一下,大概包括以下部分:

  • 文件夹的新建、删除、重命名
  • 文件夹中子文件和目录的统计
  • 文件的新建及显示文件内容
  • 文件在local和remote间的相互复制
  • 定位文件在HDFS中的位置,以及副本存放的主机
  • HDFS资源使用情况

1. 新建文件夹

public void mkdirs(String folder) throws IOException {    Path path = new Path(folder);    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);    if (!fs.exists(path)) {        fs.mkdirs(path);        System.out.println("Create: " + folder);    }    fs.close();}

 

2. 删除文件夹

public void rmr(String folder) throws IOException {    Path path = new Path(folder);    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);    fs.deleteOnExit(path);    System.out.println("Delete: " + folder);    fs.close();}

 

3. 文件重命名

public void rename(String src, String dst) throws IOException {    Path name1 = new Path(src);    Path name2 = new Path(dst);    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);    fs.rename(name1, name2);    System.out.println("Rename: from " + src + " to " + dst);    fs.close();}

 

4. 列出文件夹中的子文件及目录

public void ls(String folder) throws IOException {    Path path = new Path(folder);    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);    FileStatus[] list = fs.listStatus(path);        System.out.println("ls: " + folder);    System.out.println("==========================================================");    for (FileStatus f : list) {        System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDirectory(), f.getLen());    }    System.out.println("==========================================================");    fs.close();}

 

5. 创建文件,并添加内容

public void createFile(String file, String content) throws IOException {    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);    byte[] buff = content.getBytes();    FSDataOutputStream os = null;    try {        os = fs.create(new Path(file));        os.write(buff, 0, buff.length);        System.out.println("Create: " + file);    } finally {        if (os != null)            os.close();    }    fs.close();}

 

6. 将local数据复制到remote

public void copyFile(String local, String remote) throws IOException {    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);    fs.copyFromLocalFile(new Path(local), new Path(remote));    System.out.println("copy from: " + local + " to " + remote);    fs.close();}

 

7. 将remote数据下载到local

public void download(String remote, String local) throws IOException {    Path path = new Path(remote);    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);    fs.copyToLocalFile(path, new Path(local));    System.out.println("download: from" + remote + " to " + local);    fs.close();}

 

8. 显示文件内容

public String cat(String remoteFile) throws IOException {        Path path = new Path(remoteFile);        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        FSDataInputStream fsdis = null;        System.out.println("cat: " + remoteFile);        OutputStream baos = new ByteArrayOutputStream();        String str = null;        try {            fsdis = fs.open(path);            IOUtils.copyBytes(fsdis, baos, 4096, false);            str = baos.toString();        } finally {            IOUtils.closeStream(fsdis);            fs.close();        }        System.out.println(str);        return str;    }

 

9. 定位一个文件在HDFS中存储的位置,以及多个副本存储在集群哪些节点上

public void location() throws IOException {    String folder = hdfsPath + "create/";    String file = "t2.txt";    FileSystem fs = FileSystem.get(URI.create(hdfsPath), new Configuration());    FileStatus f = fs.getFileStatus(new Path(folder + file));    BlockLocation[] list = fs.getFileBlockLocations(f, 0, f.getLen());    System.out.println("File Location: " + folder + file);    for (BlockLocation bl : list) {        String[] hosts = bl.getHosts();        for (String host : hosts) {            System.out.println("host:" + host);        }    }    fs.close();}

 

10. 获取HDFS集群存储资源使用情况

public void getTotalCapacity() {    try {        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        FsStatus fsStatus = fs.getStatus();        System.out.println("总容量:" + fsStatus.getCapacity());        System.out.println("使用容量:" + fsStatus.getUsed());        System.out.println("剩余容量:" + fsStatus.getRemaining());    } catch (IOException e) {        e.printStackTrace();    }}

 

完整代码

import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.OutputStream;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.BlockLocation;import org.apache.hadoop.fs.ContentSummary;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.FsStatus;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.mapred.JobConf;/** HDFS工具类* */public class Hdfs {    private static final String HDFS = "hdfs://10.20.14.47:8020/";    public Hdfs(Configuration conf) {        this(HDFS, conf);    }    public Hdfs(String hdfs, Configuration conf) {        this.hdfsPath = hdfs;        this.conf = conf;    }    private String hdfsPath;    private Configuration conf;    public static void main(String[] args) throws IOException {        JobConf conf = config();        Hdfs hdfs = new Hdfs(conf);        hdfs.createFile("/create/t2.txt", "12");        hdfs.location();    }    public static JobConf config() {        JobConf conf = new JobConf(Hdfs.class);        conf.setJobName("HdfsDAO");        conf.addResource("classpath:/hadoop/core-site.xml");        conf.addResource("classpath:/hadoop/hdfs-site.xml");        conf.addResource("classpath:/hadoop/mapred-site.xml");        return conf;    }        /*     * 创建文件夹     */    public void mkdirs(String folder) throws IOException {        Path path = new Path(folder);        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        if (!fs.exists(path)) {            fs.mkdirs(path);            System.out.println("Create: " + folder);        }        fs.close();    }    /*     * 删除文件夹     */    public void rmr(String folder) throws IOException {        Path path = new Path(folder);        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        fs.deleteOnExit(path);        System.out.println("Delete: " + folder);        fs.close();    }    /*     * 文件重命名     */    public void rename(String src, String dst) throws IOException {        Path name1 = new Path(src);        Path name2 = new Path(dst);        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        fs.rename(name1, name2);        System.out.println("Rename: from " + src + " to " + dst);        fs.close();    }    /*     * 列出文件夹中的子文件及目录     */    public void ls(String folder) throws IOException {        Path path = new Path(folder);        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        FileStatus[] list = fs.listStatus(path);                System.out.println("ls: " + folder);        System.out.println("==========================================================");        for (FileStatus f : list) {            System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDirectory(), f.getLen());        }        System.out.println("==========================================================");        fs.close();    }    /*     * 创建文件,并添加内容     */    public void createFile(String file, String content) throws IOException {        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        byte[] buff = content.getBytes();        FSDataOutputStream os = null;        try {            os = fs.create(new Path(file));            os.write(buff, 0, buff.length);            System.out.println("Create: " + file);        } finally {            if (os != null)                os.close();        }        fs.close();    }    /*     * 将local的数据复制到remote     */    public void copyFile(String local, String remote) throws IOException {        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        fs.copyFromLocalFile(new Path(local), new Path(remote));        System.out.println("copy from: " + local + " to " + remote);        fs.close();    }    /*     * 将remote数据下载到local     */    public void download(String remote, String local) throws IOException {        Path path = new Path(remote);        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        fs.copyToLocalFile(path, new Path(local));        System.out.println("download: from" + remote + " to " + local);        fs.close();    }    /*     * 显示文件内容     */    public String cat(String remoteFile) throws IOException {        Path path = new Path(remoteFile);        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);        FSDataInputStream fsdis = null;        System.out.println("cat: " + remoteFile);        OutputStream baos = new ByteArrayOutputStream();        String str = null;        try {            fsdis = fs.open(path);            IOUtils.copyBytes(fsdis, baos, 4096, false);            str = baos.toString();        } finally {            IOUtils.closeStream(fsdis);            fs.close();        }        System.out.println(str);        return str;    }    /*     * 定位一个文件在HDFS中存储的位置,以及多个副本存储在集群哪些节点上     */    public void location() throws IOException {        String folder = hdfsPath + "create/";        String file = "t2.txt";        FileSystem fs = FileSystem.get(URI.create(hdfsPath), new Configuration());        FileStatus f = fs.getFileStatus(new Path(folder + file));        BlockLocation[] list = fs.getFileBlockLocations(f, 0, f.getLen());        System.out.println("File Location: " + folder + file);        for (BlockLocation bl : list) {            String[] hosts = bl.getHosts();            for (String host : hosts) {                System.out.println("host:" + host);            }        }        fs.close();    }        /*     * 获取HDFS资源使用情况     */    public void getTotalCapacity() {        try {            FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);            FsStatus fsStatus = fs.getStatus();            System.out.println("总容量:" + fsStatus.getCapacity());            System.out.println("使用容量:" + fsStatus.getUsed());            System.out.println("剩余容量:" + fsStatus.getRemaining());        } catch (IOException e) {            e.printStackTrace();        }    }        /*     * 获取某文件中包含的目录数,文件数,及占用空间大小     */    public void getContentSummary(String path) {        ContentSummary cs = null;        try {            FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);            cs = fs.getContentSummary(new Path(path));        } catch (Exception e) {            e.printStackTrace();        }        // 目录数        Long directoryCount = cs.getDirectoryCount();        // 文件数        Long fileCount = cs.getFileCount();        // 占用空间        Long length = cs.getLength();                System.out.println("目录数:" + directoryCount);        System.out.println("文件数:" + fileCount);        System.out.println("占用空间:" + length);    }}
View Code

 

转载于:https://www.cnblogs.com/walker-/p/9768834.html

你可能感兴趣的文章
如何使用USBWebserver在本机快速建立网站测试环境
查看>>
变量提升
查看>>
jquery-jqzoom 插件 用例
查看>>
查看oracle数据库的连接数以及用户
查看>>
三.野指针和free
查看>>
简单【用户输入验证】
查看>>
python tkinter GUI绘制,以及点击更新显示图片
查看>>
20130330java基础学习笔记-语句_for循环嵌套练习2
查看>>
Spring面试题
查看>>
C语言栈的实现
查看>>
TC SRM 593 DIV1 250
查看>>
SRM 628 DIV2
查看>>
2018-2019-2 20165314『网络对抗技术』Exp5:MSF基础应用
查看>>
SecureCRT的使用方法和技巧(详细使用教程)
查看>>
自建数据源(RSO2)、及数据源增强
查看>>
关于View控件中的Context选择
查看>>
2018icpc徐州OnlineA Hard to prepare
查看>>
使用命令创建数据库和表
查看>>
在16aspx.com上下了一个简单商品房销售系统源码,怎么修改它的默认登录名和密码...
查看>>
linux下Rtree的安装
查看>>