|
|
@@ -0,0 +1,376 @@
|
|
|
+//package cn.com.taiji.dataService.service.impl;
|
|
|
+//
|
|
|
+//import cn.com.taiji.dataService.model.BtxaFileBo;
|
|
|
+//import cn.com.taiji.dataService.service.IBtxaeCryptoStorageService;
|
|
|
+//import com.google.common.base.Strings;
|
|
|
+//import com.google.common.collect.Lists;
|
|
|
+//import java.io.BufferedInputStream;
|
|
|
+//import java.io.File;
|
|
|
+//import java.io.FileInputStream;
|
|
|
+//import java.io.FileNotFoundException;
|
|
|
+//import java.io.FileOutputStream;
|
|
|
+//import java.io.IOException;
|
|
|
+//import java.io.InputStream;
|
|
|
+//import java.security.PrivilegedExceptionAction;
|
|
|
+//import java.util.ArrayList;
|
|
|
+//import java.util.Iterator;
|
|
|
+//import java.util.List;
|
|
|
+//
|
|
|
+//import org.apache.hadoop.conf.Configuration;
|
|
|
+//import org.apache.hadoop.fs.*;
|
|
|
+//import org.apache.hadoop.io.IOUtils;
|
|
|
+//import org.apache.hadoop.security.UserGroupInformation;
|
|
|
+//import org.springframework.stereotype.Service;
|
|
|
+//
|
|
|
+///**
|
|
|
+// * @author Ghh
|
|
|
+// * @date 2025年03月03日 12:21
|
|
|
+// */
|
|
|
+//@Service
|
|
|
+//public class BtxaCryptoStorageServiceImpl implements IBtxaeCryptoStorageService {
|
|
|
+// public static final int ENCRYPTO_BLOCK_MIN_BYTE_SIZE = 16;
|
|
|
+// private List<String> confFilePaths = Lists.newArrayList();
|
|
|
+// private FileSystem fileSystem = null;
|
|
|
+// private String username;
|
|
|
+//
|
|
|
+// public BtxaCryptoStorageServiceImpl(String... confFilePaths) {
|
|
|
+// this.confFilePaths = Lists.newArrayList(confFilePaths);
|
|
|
+// }
|
|
|
+//
|
|
|
+// private FileSystem getFiledSystem() throws IOException {
|
|
|
+// if (null != this.fileSystem) {
|
|
|
+// return this.fileSystem;
|
|
|
+// } else {
|
|
|
+// final Configuration conf = new Configuration();
|
|
|
+// Iterator var2 = this.confFilePaths.iterator();
|
|
|
+//
|
|
|
+// while(var2.hasNext()) {
|
|
|
+// String filePath = (String)var2.next();
|
|
|
+// File file = new File(filePath);
|
|
|
+// if (file.exists()) {
|
|
|
+// conf.addResource(file.toURL());
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// if (Strings.isNullOrEmpty(this.username)) {
|
|
|
+// this.fileSystem = FileSystem.get(conf);
|
|
|
+// } else {
|
|
|
+// UserGroupInformation ugi = UserGroupInformation.createRemoteUser(this.username);
|
|
|
+// try {
|
|
|
+// this.fileSystem = (FileSystem)ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
|
|
|
+// @Override
|
|
|
+// public FileSystem run() throws Exception {
|
|
|
+// return FileSystem.get(conf);
|
|
|
+// }
|
|
|
+// });
|
|
|
+// } catch (InterruptedException var5) {
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// return this.fileSystem;
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public FileSystem login(String username, String passwd) throws IOException {
|
|
|
+// this.username = username;
|
|
|
+// FileSystem filedSystem = this.getFiledSystem();
|
|
|
+// return filedSystem;
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public String read(String hdfsPath, String localPath) throws IOException {
|
|
|
+// Path path = new Path(hdfsPath);
|
|
|
+// boolean exists = this.getFiledSystem().exists(path);
|
|
|
+// if (!exists) {
|
|
|
+// throw new FileNotFoundException("hdfs filesystem file is not exists.path:" + hdfsPath);
|
|
|
+// } else {
|
|
|
+// boolean isFile = this.getFiledSystem().isFile(path);
|
|
|
+// if (!isFile) {
|
|
|
+// throw new IllegalArgumentException("hdfs filesystem path is not a file,cannot support not a file read.path:" + hdfsPath);
|
|
|
+// } else {
|
|
|
+// File localFile = new File(localPath);
|
|
|
+// if (localFile.exists()) {
|
|
|
+// if (localFile.isFile()) {
|
|
|
+// throw new PathExistsException("local filesystem path exists.path:" + localPath);
|
|
|
+// }
|
|
|
+//
|
|
|
+// localPath = localPath + "/" + path.getName();
|
|
|
+// localFile = new File(localPath);
|
|
|
+// }
|
|
|
+//
|
|
|
+// boolean newFile = localFile.createNewFile();
|
|
|
+// if (!newFile) {
|
|
|
+// throw new PathAccessDeniedException("cannot create new local file:" + localFile.getAbsolutePath());
|
|
|
+// } else {
|
|
|
+// FSDataInputStream fsDataInputStream = null;
|
|
|
+// FileOutputStream fileOutputStream = null;
|
|
|
+// boolean var10 = false;
|
|
|
+//
|
|
|
+// try {
|
|
|
+// fsDataInputStream = this.getFiledSystem().open(path);
|
|
|
+// fileOutputStream = new FileOutputStream(localFile);
|
|
|
+// IOUtils.copyBytes(fsDataInputStream, fileOutputStream, 4096, false);
|
|
|
+// } catch (IOException var15) {
|
|
|
+// } finally {
|
|
|
+// IOUtils.closeStream(fileOutputStream);
|
|
|
+// IOUtils.closeStream(fsDataInputStream);
|
|
|
+// return localPath;
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// private boolean isAllEmptyBytes(byte[] bytes) {
|
|
|
+// byte zero = bytes[0];
|
|
|
+// int i = 1;
|
|
|
+//
|
|
|
+// for(int len = bytes.length; i < len; ++i) {
|
|
|
+// if (zero != bytes[i]) {
|
|
|
+// return false;
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// return true;
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public String write(String localPath, String hdfsPath) throws Exception {
|
|
|
+// File file = new File(localPath);
|
|
|
+// if (!file.exists()) {
|
|
|
+// throw new PathExistsException("local filesystem path exists.path:" + localPath);
|
|
|
+// } else if (!file.isFile()) {
|
|
|
+// throw new FileNotFoundException("local filesystem path is directory,cannot support directory.path:" + localPath);
|
|
|
+// } else {
|
|
|
+// hdfsPath = this.getCheckPath(file.getName(), hdfsPath);
|
|
|
+// Path hdfsFilePath = new Path(hdfsPath);
|
|
|
+// FileInputStream fileInputStream = null;
|
|
|
+// BufferedInputStream bufferedInputStream = null;
|
|
|
+// String var7;
|
|
|
+// try {
|
|
|
+// fileInputStream = new FileInputStream(file);
|
|
|
+// bufferedInputStream = new BufferedInputStream(fileInputStream);
|
|
|
+// this.write((InputStream)bufferedInputStream, (Path)hdfsFilePath);
|
|
|
+// var7 = hdfsPath;
|
|
|
+// } catch (Throwable var12) {
|
|
|
+// boolean delete = this.getFiledSystem().delete(hdfsFilePath, false);
|
|
|
+// throw var12;
|
|
|
+// } finally {
|
|
|
+// IOUtils.closeStream(bufferedInputStream);
|
|
|
+// IOUtils.closeStream(fileInputStream);
|
|
|
+// }
|
|
|
+// return var7;
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// private String getCheckPath(String name, String hdfsPath) throws IOException {
|
|
|
+// Path hdfsFilePath = new Path(hdfsPath);
|
|
|
+// boolean hdfsPathExists = this.getFiledSystem().exists(hdfsFilePath);
|
|
|
+// boolean isDirDdfsPath = this.getFiledSystem().isDirectory(hdfsFilePath);
|
|
|
+// if (hdfsPathExists) {
|
|
|
+// if (!isDirDdfsPath) {
|
|
|
+// String log = "hdfs path exists.path:" + hdfsPath;
|
|
|
+// throw new PathExistsException(log);
|
|
|
+// }
|
|
|
+//
|
|
|
+// hdfsPath = hdfsPath + File.separator + name;
|
|
|
+// }
|
|
|
+//
|
|
|
+// return hdfsPath;
|
|
|
+// }
|
|
|
+//
|
|
|
+// private void write(InputStream localStream, Path hdfsPath) throws IOException {
|
|
|
+// FSDataOutputStream outputStream = null;
|
|
|
+// try {
|
|
|
+// outputStream = this.getFiledSystem().create(hdfsPath);
|
|
|
+// int buffSize = 4096;
|
|
|
+// byte[] buf = new byte[buffSize];
|
|
|
+// int bytesRead = localStream.read(buf);
|
|
|
+//
|
|
|
+// for(int count = 0; bytesRead >= 0; bytesRead = localStream.read(buf)) {
|
|
|
+// count += bytesRead;
|
|
|
+// outputStream.write(buf, 0, bytesRead);
|
|
|
+// }
|
|
|
+// } finally {
|
|
|
+// IOUtils.closeStream(outputStream);
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public boolean mkdir(String path) {
|
|
|
+// try {
|
|
|
+// return this.getFiledSystem().mkdirs(new Path(path));
|
|
|
+// } catch (IOException var3) {
|
|
|
+// return false;
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public boolean rm(String path, boolean isRecursive) {
|
|
|
+// try {
|
|
|
+// return this.getFiledSystem().delete(new Path(path), isRecursive);
|
|
|
+// } catch (IOException var4) {
|
|
|
+// return false;
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public List<BtxaFileBo> ls(String path) {
|
|
|
+// ArrayList files = Lists.newArrayList();
|
|
|
+//
|
|
|
+// try {
|
|
|
+// RemoteIterator iterator = this.getFiledSystem().listLocatedStatus(new Path(path));
|
|
|
+//
|
|
|
+// while(iterator.hasNext()) {
|
|
|
+// BtxaFileBo btxaFileBo = new BtxaFileBo();
|
|
|
+// LocatedFileStatus fileStatus = (LocatedFileStatus)iterator.next();
|
|
|
+// btxaFileBo.setFile(fileStatus.isFile());
|
|
|
+// btxaFileBo.setPath(fileStatus.getPath().toUri().getPath());
|
|
|
+// btxaFileBo.setName(fileStatus.getPath().getName());
|
|
|
+// files.add(btxaFileBo);
|
|
|
+// }
|
|
|
+// } catch (IOException var6) {
|
|
|
+// }
|
|
|
+//
|
|
|
+// return files;
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public List<BtxaFileBo> lsr(String path) {
|
|
|
+// ArrayList files = Lists.newArrayList();
|
|
|
+//
|
|
|
+// try {
|
|
|
+// RemoteIterator iterator = this.getFiledSystem().listFiles(new Path(path), true);
|
|
|
+//
|
|
|
+// while(iterator.hasNext()) {
|
|
|
+// BtxaFileBo btxaFileBo = new BtxaFileBo();
|
|
|
+// LocatedFileStatus fileStatus = (LocatedFileStatus)iterator.next();
|
|
|
+// btxaFileBo.setFile(fileStatus.isFile());
|
|
|
+// btxaFileBo.setPath(fileStatus.getPath().toUri().getPath());
|
|
|
+// btxaFileBo.setName(fileStatus.getPath().getName());
|
|
|
+// files.add(btxaFileBo);
|
|
|
+// }
|
|
|
+// } catch (IOException var6) {
|
|
|
+// }
|
|
|
+//
|
|
|
+// return files;
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public void destroy() {
|
|
|
+// try {
|
|
|
+// this.fileSystem.close();
|
|
|
+// } catch (IOException var2) {
|
|
|
+// var2.printStackTrace();
|
|
|
+// }
|
|
|
+//
|
|
|
+// }
|
|
|
+//
|
|
|
+// public static void main(String[] args) throws Exception {
|
|
|
+// String username = args[0];
|
|
|
+// String methodName = args[1];
|
|
|
+// List<String> confs = Lists.newArrayList(new String[]{"/home/hadoop/hadoop-3.2.1/etc/hadoop/core-site.xml","/home/hadoop/hadoop-3.2.1/etc/hadoop/hdfs-site.xml"});
|
|
|
+// BtxaCryptoStorageServiceImpl btxaCryptoStorageService = new BtxaCryptoStorageServiceImpl((String[])confs.toArray(new String[0]));
|
|
|
+// btxaCryptoStorageService.login(username, username);
|
|
|
+// String lookDir = null;
|
|
|
+// byte var7 = -1;
|
|
|
+// switch(methodName.hashCode()) {
|
|
|
+// case 3463:
|
|
|
+// if (methodName.equals("ls")) {
|
|
|
+// var7 = 4;
|
|
|
+// }
|
|
|
+// break;
|
|
|
+// case 3643:
|
|
|
+// if (methodName.equals("rm")) {
|
|
|
+// var7 = 3;
|
|
|
+// }
|
|
|
+// break;
|
|
|
+// case 107467:
|
|
|
+// if (methodName.equals("lsr")) {
|
|
|
+// var7 = 5;
|
|
|
+// }
|
|
|
+// break;
|
|
|
+// case 3496342:
|
|
|
+// if (methodName.equals("read")) {
|
|
|
+// var7 = 0;
|
|
|
+// }
|
|
|
+// break;
|
|
|
+// case 103950895:
|
|
|
+// if (methodName.equals("mkdir")) {
|
|
|
+// var7 = 2;
|
|
|
+// }
|
|
|
+// break;
|
|
|
+// case 113399775:
|
|
|
+// if (methodName.equals("write")) {
|
|
|
+// var7 = 1;
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// switch(var7) {
|
|
|
+// case 0:
|
|
|
+// btxaCryptoStorageService.read(args[2], args[3]);
|
|
|
+// System.out.println(String.format(">>>>>>read remote file:%s to local file:%s success.", args[2], args[3]));
|
|
|
+// break;
|
|
|
+// case 1:
|
|
|
+// String write = btxaCryptoStorageService.write(args[2], args[3]);
|
|
|
+// System.out.println(String.format(">>>>>>upload local file:%s to remote file:%s success.", args[2], args[3]));
|
|
|
+// if (!write.equals(args[2])) {
|
|
|
+// System.out.println(">>>>>> !!! upload remote file name change to " + write);
|
|
|
+// }
|
|
|
+//
|
|
|
+// lookDir = args[3].substring(0, args[3].lastIndexOf("/") + 1);
|
|
|
+// break;
|
|
|
+// case 2:
|
|
|
+// boolean mkdir = btxaCryptoStorageService.mkdir(args[2]);
|
|
|
+// System.out.println(">>>>>>create dir is " + (mkdir ? "success" : "error"));
|
|
|
+// lookDir = args[2].substring(0, args[2].lastIndexOf("/") + 1);
|
|
|
+// break;
|
|
|
+// case 3:
|
|
|
+// boolean rm = btxaCryptoStorageService.rm(args[2], Boolean.getBoolean(args[3]));
|
|
|
+// System.out.println(">>>>>>delete file or dir is " + (rm ? "success" : "error"));
|
|
|
+// lookDir = args[2].substring(0, args[2].lastIndexOf("/") + 1);
|
|
|
+// break;
|
|
|
+// case 4:
|
|
|
+// System.out.println(">>>>>>start look dir:" + args[2]);
|
|
|
+// List<BtxaFileBo> ls = btxaCryptoStorageService.ls(args[2]);
|
|
|
+// Iterator var17 = ls.iterator();
|
|
|
+//
|
|
|
+// while(var17.hasNext()) {
|
|
|
+// BtxaFileBo file = (BtxaFileBo)var17.next();
|
|
|
+// System.out.println(file.getPath());
|
|
|
+// }
|
|
|
+//
|
|
|
+// System.out.println(">>>>>>end look dir:");
|
|
|
+// break;
|
|
|
+// case 5:
|
|
|
+// List<BtxaFileBo> lsr = btxaCryptoStorageService.lsr(args[2]);
|
|
|
+// System.out.println(">>>>>>start look dir all file and dir:");
|
|
|
+// Iterator var14 = lsr.iterator();
|
|
|
+//
|
|
|
+// while(var14.hasNext()) {
|
|
|
+// BtxaFileBo file = (BtxaFileBo)var14.next();
|
|
|
+// System.out.println(file.getPath());
|
|
|
+// }
|
|
|
+//
|
|
|
+// System.out.println(">>>>>>end look dir all file and dir.");
|
|
|
+// }
|
|
|
+//
|
|
|
+// System.out.println(">>>>>>run method " + methodName + " is done.");
|
|
|
+// if (null != lookDir) {
|
|
|
+// System.out.println(">>>>>>start look dir:" + lookDir);
|
|
|
+// List<BtxaFileBo> ls = btxaCryptoStorageService.lsr(lookDir);
|
|
|
+// Iterator var16 = ls.iterator();
|
|
|
+//
|
|
|
+// while(var16.hasNext()) {
|
|
|
+// BtxaFileBo file = (BtxaFileBo)var16.next();
|
|
|
+// System.out.println(file.getPath());
|
|
|
+// }
|
|
|
+//
|
|
|
+// System.out.println(">>>>>>end look dir:");
|
|
|
+// }
|
|
|
+//
|
|
|
+// btxaCryptoStorageService.destroy();
|
|
|
+// System.exit(0);
|
|
|
+// }
|
|
|
+//}
|
guoyuanquan@mail.taiji.com.cn