好久没写spark程序了,今天编写了一个spark程序,但是执行的时候报错了,提示:
org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z
这个一看是hadoop操作文件的报错,后来在网上查了一下,提示修改下NativeIO这个类就可以了。因此这里主要记录一下,具体的修改后的NativeIO类的内容如下:
// // Source code recreated from a .class file by IntelliJ IDEA // (powered by FernFlower decompiler) // package org.apache.hadoop.io.nativeio; import java.io.Closeable; import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.HardLink; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.util.CleanerUtil; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.PerformanceAdvisory; import org.apache.hadoop.util.Shell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sun.misc.Unsafe; @Private @Unstable public class NativeIO { private static boolean workaroundNonThreadSafePasswdCalls = false; private static final Logger LOG = LoggerFactory.getLogger(NativeIO.class); private static boolean nativeLoaded = false; private static final Map<Long, NativeIO.CachedUid> uidCache; private static long cacheTimeout; private static boolean initialized; public NativeIO() { } public static boolean isAvailable() { return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded; } private static native void initNative(); static long getMemlockLimit() { return isAvailable() ? getMemlockLimit0() : 0L; } private static native long getMemlockLimit0(); static long getOperatingSystemPageSize() { try { Field f = Unsafe.class.getDeclaredField("theUnsafe"); f.setAccessible(true); Unsafe unsafe = (Unsafe)f.get((Object)null); return (long)unsafe.pageSize(); } catch (Throwable var2) { LOG.warn("Unable to get operating system page size. Guessing 4096.", var2); return 4096L; } } private static String stripDomain(String name) { int i = name.indexOf(92); if (i != -1) { name = name.substring(i + 1); } return name; } public static String getOwner(FileDescriptor fd) throws IOException { ensureInitialized(); if (Shell.WINDOWS) { String owner = NativeIO.Windows.getOwner(fd); owner = stripDomain(owner); return owner; } else { long uid = NativeIO.POSIX.getUIDforFDOwnerforOwner(fd); NativeIO.CachedUid cUid = (NativeIO.CachedUid)uidCache.get(uid); long now = System.currentTimeMillis(); if (cUid != null && cUid.timestamp + cacheTimeout > now) { return cUid.username; } else { String user = NativeIO.POSIX.getUserName(uid); LOG.info("Got UserName " + user + " for UID " + uid + " from the native implementation"); cUid = new NativeIO.CachedUid(user, now); uidCache.put(uid, cUid); return user; } } } public static FileDescriptor getShareDeleteFileDescriptor(File f, long seekOffset) throws IOException { if (!Shell.WINDOWS) { RandomAccessFile rf = new RandomAccessFile(f, "r"); if (seekOffset > 0L) { rf.seek(seekOffset); } return rf.getFD(); } else { FileDescriptor fd = NativeIO.Windows.createFile(f.getAbsolutePath(), 2147483648L, 7L, 3L); if (seekOffset > 0L) { NativeIO.Windows.setFilePointer(fd, seekOffset, 0L); } return fd; } } public static FileOutputStream getCreateForWriteFileOutputStream(File f, int permissions) throws IOException { FileDescriptor fd; if (!Shell.WINDOWS) { try { fd = NativeIO.POSIX.open(f.getAbsolutePath(), NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT | NativeIO.POSIX.O_EXCL, permissions); return new FileOutputStream(fd); } catch (NativeIOException var4) { if (var4.getErrno() == Errno.EEXIST) { throw new AlreadyExistsException(var4); } else { throw var4; } } } else { try { fd = NativeIO.Windows.createFile(f.getCanonicalPath(), 1073741824L, 7L, 1L); NativeIO.POSIX.chmod(f.getCanonicalPath(), permissions); return new FileOutputStream(fd); } catch (NativeIOException var3) { if (var3.getErrorCode() == 80L) { throw new AlreadyExistsException(var3); } else { throw var3; } } } } private static synchronized void ensureInitialized() { if (!initialized) { cacheTimeout = (new Configuration()).getLong("hadoop.security.uid.cache.secs", 14400L) * 1000L; LOG.info("Initialized cache for UID to User mapping with a cache timeout of " + cacheTimeout / 1000L + " seconds."); initialized = true; } } public static void renameTo(File src, File dst) throws IOException { if (!nativeLoaded) { if (!src.renameTo(dst)) { throw new IOException("renameTo(src=" + src + ", dst=" + dst + ") failed."); } } else { renameTo0(src.getAbsolutePath(), dst.getAbsolutePath()); } } /** @deprecated */ @Deprecated public static void link(File src, File dst) throws IOException { if (!nativeLoaded) { HardLink.createHardLink(src, dst); } else { link0(src.getAbsolutePath(), dst.getAbsolutePath()); } } private static native void renameTo0(String var0, String var1) throws NativeIOException; private static native void link0(String var0, String var1) throws NativeIOException; public static void copyFileUnbuffered(File src, File dst) throws IOException { if (nativeLoaded && Shell.WINDOWS) { copyFileUnbuffered0(src.getAbsolutePath(), dst.getAbsolutePath()); } else { FileInputStream fis = new FileInputStream(src); FileChannel input = null; try { input = fis.getChannel(); FileOutputStream fos = new FileOutputStream(dst); Throwable var5 = null; try { FileChannel output = fos.getChannel(); Throwable var7 = null; try { long remaining = input.size(); long position = 0L; for(long transferred = 0L; remaining > 0L; position += transferred) { transferred = input.transferTo(position, remaining, output); remaining -= transferred; } } catch (Throwable var47) { var7 = var47; throw var47; } finally { if (output != null) { if (var7 != null) { try { output.close(); } catch (Throwable var46) { var7.addSuppressed(var46); } } else { output.close(); } } } } catch (Throwable var49) { var5 = var49; throw var49; } finally { if (fos != null) { if (var5 != null) { try { fos.close(); } catch (Throwable var45) { var5.addSuppressed(var45); } } else { fos.close(); } } } } finally { IOUtils.cleanupWithLogger(LOG, new Closeable[]{input, fis}); } } } private static native void copyFileUnbuffered0(String var0, String var1) throws NativeIOException; static { if (NativeCodeLoader.isNativeCodeLoaded()) { try { initNative(); nativeLoaded = true; } catch (Throwable var1) { PerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", var1); } } uidCache = new ConcurrentHashMap(); initialized = false; } private static class CachedUid { final long timestamp; final String username; public CachedUid(String username, long timestamp) { this.timestamp = timestamp; this.username = username; } } public static class Windows { public static final long GENERIC_READ = 2147483648L; public static final long GENERIC_WRITE = 1073741824L; public static final long FILE_SHARE_READ = 1L; public static final long FILE_SHARE_WRITE = 2L; public static final long FILE_SHARE_DELETE = 4L; public static final long CREATE_NEW = 1L; public static final long CREATE_ALWAYS = 2L; public static final long OPEN_EXISTING = 3L; public static final long OPEN_ALWAYS = 4L; public static final long TRUNCATE_EXISTING = 5L; public static final long FILE_BEGIN = 0L; public static final long FILE_CURRENT = 1L; public static final long FILE_END = 2L; public static final long FILE_ATTRIBUTE_NORMAL = 128L; public Windows() { } public static void createDirectoryWithMode(File path, int mode) throws IOException { createDirectoryWithMode0(path.getAbsolutePath(), mode); } private static native void createDirectoryWithMode0(String var0, int var1) throws NativeIOException; public static native FileDescriptor createFile(String var0, long var1, long var3, long var5) throws IOException; public static FileOutputStream createFileOutputStreamWithMode(File path, boolean append, int mode) throws IOException { long desiredAccess = 1073741824L; long shareMode = 3L; long creationDisposition = append ? 4L : 2L; return new FileOutputStream(createFileWithMode0(path.getAbsolutePath(), desiredAccess, shareMode, creationDisposition, mode)); } private static native FileDescriptor createFileWithMode0(String var0, long var1, long var3, long var5, int var7) throws NativeIOException; public static native long setFilePointer(FileDescriptor var0, long var1, long var3) throws IOException; private static native String getOwner(FileDescriptor var0) throws IOException; private static native boolean access0(String var0, int var1); public static boolean access(String path, NativeIO.Windows.AccessRight desiredAccess) throws IOException { //return access0(path, desiredAccess.accessRight()); return true; } public static native void extendWorkingSetSize(long var0) throws IOException; static { if (NativeCodeLoader.isNativeCodeLoaded()) { try { NativeIO.initNative(); NativeIO.nativeLoaded = true; } catch (Throwable var1) { PerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", var1); } } } public static enum AccessRight { ACCESS_READ(1), ACCESS_WRITE(2), ACCESS_EXECUTE(32); private final int accessRight; private AccessRight(int access) { this.accessRight = access; } public int accessRight() { return this.accessRight; } } } public static class POSIX { public static int O_RDONLY = -1; public static int O_WRONLY = -1; public static int O_RDWR = -1; public static int O_CREAT = -1; public static int O_EXCL = -1; public static int O_NOCTTY = -1; public static int O_TRUNC = -1; public static int O_APPEND = -1; public static int O_NONBLOCK = -1; public static int O_SYNC = -1; public static int POSIX_FADV_NORMAL = -1; public static int POSIX_FADV_RANDOM = -1; public static int POSIX_FADV_SEQUENTIAL = -1; public static int POSIX_FADV_WILLNEED = -1; public static int POSIX_FADV_DONTNEED = -1; public static int POSIX_FADV_NOREUSE = -1; public static int SYNC_FILE_RANGE_WAIT_BEFORE = 1; public static int SYNC_FILE_RANGE_WRITE = 2; public static int SYNC_FILE_RANGE_WAIT_AFTER = 4; private static NativeIO.POSIX.SupportState pmdkSupportState; private static final Logger LOG; public static boolean fadvisePossible; private static boolean nativeLoaded; private static boolean syncFileRangePossible; static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY = "hadoop.workaround.non.threadsafe.getpwuid"; static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = true; private static long cacheTimeout; private static NativeIO.POSIX.CacheManipulator cacheManipulator; private static final Map<Integer, NativeIO.POSIX.CachedName> USER_ID_NAME_CACHE; private static final Map<Integer, NativeIO.POSIX.CachedName> GROUP_ID_NAME_CACHE; public static final int MMAP_PROT_READ = 1; public static final int MMAP_PROT_WRITE = 2; public static final int MMAP_PROT_EXEC = 4; public POSIX() { } public static NativeIO.POSIX.CacheManipulator getCacheManipulator() { return cacheManipulator; } public static void setCacheManipulator(NativeIO.POSIX.CacheManipulator cacheManipulator) { NativeIO.POSIX.cacheManipulator = cacheManipulator; } public static void setPmdkSupportState(int stateCode) { NativeIO.POSIX.SupportState[] var1 = NativeIO.POSIX.SupportState.values(); int var2 = var1.length; for(int var3 = 0; var3 < var2; ++var3) { NativeIO.POSIX.SupportState state = var1[var3]; if (state.getStateCode() == stateCode) { pmdkSupportState = state; return; } } LOG.error("The state code: " + stateCode + " is unrecognized!"); } public static String getPmdkSupportStateMessage() { return getPmdkLibPath() != null ? pmdkSupportState.getMessage() + " The pmdk lib path: " + getPmdkLibPath() : pmdkSupportState.getMessage(); } public static boolean isPmdkAvailable() { LOG.info(pmdkSupportState.getMessage()); return pmdkSupportState == NativeIO.POSIX.SupportState.SUPPORTED; } private static native String getPmdkLibPath(); private static native boolean isPmemCheck(long var0, long var2); private static native NativeIO.POSIX.PmemMappedRegion pmemMapFile(String var0, long var1, boolean var3); private static native boolean pmemUnMap(long var0, long var2); private static native void pmemCopy(byte[] var0, long var1, boolean var3, long var4); private static native void pmemDrain(); private static native void pmemSync(long var0, long var2); public static boolean isAvailable() { return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded; } private static void assertCodeLoaded() throws IOException { if (!isAvailable()) { throw new IOException("NativeIO was not loaded"); } } public static native FileDescriptor open(String var0, int var1, int var2) throws IOException; private static native NativeIO.POSIX.Stat fstat(FileDescriptor var0) throws IOException; private static native NativeIO.POSIX.Stat stat(String var0) throws IOException; private static native void chmodImpl(String var0, int var1) throws IOException; public static void chmod(String path, int mode) throws IOException { if (!Shell.WINDOWS) { chmodImpl(path, mode); } else { try { chmodImpl(path, mode); } catch (NativeIOException var3) { if (var3.getErrorCode() == 3L) { throw new NativeIOException("No such file or directory", Errno.ENOENT); } LOG.warn(String.format("NativeIO.chmod error (%d): %s", var3.getErrorCode(), var3.getMessage())); throw new NativeIOException("Unknown error", Errno.UNKNOWN); } } } static native void posix_fadvise(FileDescriptor var0, long var1, long var3, int var5) throws NativeIOException; static native void sync_file_range(FileDescriptor var0, long var1, long var3, int var5) throws NativeIOException; static void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags) throws NativeIOException { if (nativeLoaded && fadvisePossible) { try { posix_fadvise(fd, offset, len, flags); } catch (UnsatisfiedLinkError var8) { fadvisePossible = false; } } } public static void syncFileRangeIfPossible(FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException { if (nativeLoaded && syncFileRangePossible) { try { sync_file_range(fd, offset, nbytes, flags); } catch (UnsupportedOperationException var7) { syncFileRangePossible = false; } catch (UnsatisfiedLinkError var8) { syncFileRangePossible = false; } } } static native void mlock_native(ByteBuffer var0, long var1) throws NativeIOException; static void mlock(ByteBuffer buffer, long len) throws IOException { assertCodeLoaded(); if (!buffer.isDirect()) { throw new IOException("Cannot mlock a non-direct ByteBuffer"); } else { mlock_native(buffer, len); } } public static void munmap(MappedByteBuffer buffer) { if (CleanerUtil.UNMAP_SUPPORTED) { try { CleanerUtil.getCleaner().freeBuffer(buffer); } catch (IOException var2) { LOG.info("Failed to unmap the buffer", var2); } } else { LOG.trace(CleanerUtil.UNMAP_NOT_SUPPORTED_REASON); } } private static native long getUIDforFDOwnerforOwner(FileDescriptor var0) throws IOException; private static native String getUserName(long var0) throws IOException; public static NativeIO.POSIX.Stat getFstat(FileDescriptor fd) throws IOException { NativeIO.POSIX.Stat stat = null; if (!Shell.WINDOWS) { stat = fstat(fd); stat.owner = getName(NativeIO.POSIX.IdCache.USER, stat.ownerId); stat.group = getName(NativeIO.POSIX.IdCache.GROUP, stat.groupId); } else { try { stat = fstat(fd); } catch (NativeIOException var3) { if (var3.getErrorCode() == 6L) { throw new NativeIOException("The handle is invalid.", Errno.EBADF); } LOG.warn(String.format("NativeIO.getFstat error (%d): %s", var3.getErrorCode(), var3.getMessage())); throw new NativeIOException("Unknown error", Errno.UNKNOWN); } } return stat; } public static NativeIO.POSIX.Stat getStat(String path) throws IOException { if (path == null) { String errMessage = "Path is null"; LOG.warn(errMessage); throw new IOException(errMessage); } else { NativeIO.POSIX.Stat stat = null; try { if (!Shell.WINDOWS) { stat = stat(path); stat.owner = getName(NativeIO.POSIX.IdCache.USER, stat.ownerId); stat.group = getName(NativeIO.POSIX.IdCache.GROUP, stat.groupId); } else { stat = stat(path); } return stat; } catch (NativeIOException var3) { LOG.warn("NativeIO.getStat error ({}): {} -- file path: {}", new Object[]{var3.getErrorCode(), var3.getMessage(), path}); throw new PathIOException(path, var3); } } } private static String getName(NativeIO.POSIX.IdCache domain, int id) throws IOException { Map<Integer, NativeIO.POSIX.CachedName> idNameCache = domain == NativeIO.POSIX.IdCache.USER ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE; NativeIO.POSIX.CachedName cachedName = (NativeIO.POSIX.CachedName)idNameCache.get(id); long now = System.currentTimeMillis(); String name; if (cachedName != null && cachedName.timestamp + cacheTimeout > now) { name = cachedName.name; } else { name = domain == NativeIO.POSIX.IdCache.USER ? getUserName(id) : getGroupName(id); if (LOG.isDebugEnabled()) { String type = domain == NativeIO.POSIX.IdCache.USER ? "UserName" : "GroupName"; LOG.debug("Got " + type + " " + name + " for ID " + id + " from the native implementation"); } cachedName = new NativeIO.POSIX.CachedName(name, now); idNameCache.put(id, cachedName); } return name; } static native String getUserName(int var0) throws IOException; static native String getGroupName(int var0) throws IOException; public static native long mmap(FileDescriptor var0, int var1, boolean var2, long var3) throws IOException; public static native void munmap(long var0, long var2) throws IOException; static { pmdkSupportState = NativeIO.POSIX.SupportState.UNSUPPORTED; LOG = LoggerFactory.getLogger(NativeIO.class); fadvisePossible = false; nativeLoaded = false; syncFileRangePossible = true; cacheTimeout = -1L; cacheManipulator = new NativeIO.POSIX.CacheManipulator(); if (NativeCodeLoader.isNativeCodeLoaded()) { try { Configuration conf = new Configuration(); NativeIO.workaroundNonThreadSafePasswdCalls = conf.getBoolean("hadoop.workaround.non.threadsafe.getpwuid", true); NativeIO.initNative(); nativeLoaded = true; cacheTimeout = conf.getLong("hadoop.security.uid.cache.secs", 14400L) * 1000L; LOG.debug("Initialized cache for IDs to User/Group mapping with a cache timeout of " + cacheTimeout / 1000L + " seconds."); } catch (Throwable var1) { PerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", var1); } } USER_ID_NAME_CACHE = new ConcurrentHashMap(); GROUP_ID_NAME_CACHE = new ConcurrentHashMap(); } private static enum IdCache { USER, GROUP; private IdCache() { } } private static class CachedName { final long timestamp; final String name; public CachedName(String name, long timestamp) { this.name = name; this.timestamp = timestamp; } } public static class Stat { private int ownerId; private int groupId; private String owner; private String group; private int mode; public static int S_IFMT = -1; public static int S_IFIFO = -1; public static int S_IFCHR = -1; public static int S_IFDIR = -1; public static int S_IFBLK = -1; public static int S_IFREG = -1; public static int S_IFLNK = -1; public static int S_IFSOCK = -1; public static int S_ISUID = -1; public static int S_ISGID = -1; public static int S_ISVTX = -1; public static int S_IRUSR = -1; public static int S_IWUSR = -1; public static int S_IXUSR = -1; Stat(int ownerId, int groupId, int mode) { this.ownerId = ownerId; this.groupId = groupId; this.mode = mode; } Stat(String owner, String group, int mode) { if (!Shell.WINDOWS) { this.owner = owner; } else { this.owner = NativeIO.stripDomain(owner); } if (!Shell.WINDOWS) { this.group = group; } else { this.group = NativeIO.stripDomain(group); } this.mode = mode; } public String toString() { return "Stat(owner='" + this.owner + "', group='" + this.group + "', mode=" + this.mode + ")"; } public String getOwner() { return this.owner; } public String getGroup() { return this.group; } public int getMode() { return this.mode; } } @VisibleForTesting public static class NoMlockCacheManipulator extends NativeIO.POSIX.CacheManipulator { public NoMlockCacheManipulator() { } public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException { NativeIO.POSIX.LOG.info("mlocking " + identifier); } public long getMemlockLimit() { return 1125899906842624L; } public long getOperatingSystemPageSize() { return 4096L; } public boolean verifyCanMlock() { return true; } } @VisibleForTesting public static class CacheManipulator { public CacheManipulator() { } public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException { NativeIO.POSIX.mlock(buffer, len); } public long getMemlockLimit() { return NativeIO.getMemlockLimit(); } public long getOperatingSystemPageSize() { return NativeIO.getOperatingSystemPageSize(); } public void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags) throws NativeIOException { NativeIO.POSIX.posixFadviseIfPossible(identifier, fd, offset, len, flags); } public boolean verifyCanMlock() { return NativeIO.isAvailable(); } } public static class Pmem { public Pmem() { } public static boolean isPmem(long address, long length) { return NativeIO.POSIX.isPmemCheck(address, length); } public static NativeIO.POSIX.PmemMappedRegion mapBlock(String path, long length, boolean isFileExist) { return NativeIO.POSIX.pmemMapFile(path, length, isFileExist); } public static boolean unmapBlock(long address, long length) { return NativeIO.POSIX.pmemUnMap(address, length); } public static void memCopy(byte[] src, long dest, boolean isPmem, long length) { NativeIO.POSIX.pmemCopy(src, dest, isPmem, length); } public static void memSync(NativeIO.POSIX.PmemMappedRegion region) { if (region.isPmem()) { NativeIO.POSIX.pmemDrain(); } else { NativeIO.POSIX.pmemSync(region.getAddress(), region.getLength()); } } public static String getPmdkLibPath() { return NativeIO.POSIX.getPmdkLibPath(); } } public static class PmemMappedRegion { private long address; private long length; private boolean isPmem; public PmemMappedRegion(long address, long length, boolean isPmem) { this.address = address; this.length = length; this.isPmem = isPmem; } public boolean isPmem() { return this.isPmem; } public long getAddress() { return this.address; } public long getLength() { return this.length; } } public static enum SupportState { UNSUPPORTED(-1), PMDK_LIB_NOT_FOUND(1), SUPPORTED(0); private byte stateCode; private SupportState(int stateCode) { this.stateCode = (byte)stateCode; } public int getStateCode() { return this.stateCode; } public String getMessage() { String msg; switch(this.stateCode) { case -1: msg = "The native code was built without PMDK support."; break; case 0: msg = "The native code was built with PMDK support, and PMDK libs were loaded successfully."; break; case 1: msg = "The native code was built with PMDK support, but PMDK libs were NOT found in execution environment or failed to be loaded."; break; default: msg = "The state code: " + this.stateCode + " is unrecognized!"; } return msg; } } } }
在这里也提供NativeIO的类下载,点击下载
把这个类放在当前的工程项目上,记得包名也需要和这个类的原始包名保持一致。如图:
还没有评论,来说两句吧...