IDEA MAPREDUCE开发无法从HDFS输入并输出？

来源：4-7 词频统计之自定义Driver类实现

begin_0002

2019-07-31

DEBUG] method:org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback.(JniBasedUnixGroupsMappingWithFallback.java:45)
Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping
[DEBUG] method:org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:372)
Failed to detect a valid hadoop home directory
java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set.
at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:354)
at org.apache.hadoop.util.Shell.(Shell.java:379)
at org.apache.hadoop.util.StringUtils.(StringUtils.java:79)
at org.apache.hadoop.security.Groups.parseStaticMapping(Groups.java:168)
at org.apache.hadoop.security.Groups.(Groups.java:132)
at org.apache.hadoop.security.Groups.(Groups.java:100)
at org.apache.hadoop.security.Groups.getUserToGroupsMappingService(Groups.java:435)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:341)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:308)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:895)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:861)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:728)
at org.apache.hadoop.mapreduce.task.JobContextImpl.(JobContextImpl.java:72)
at org.apache.hadoop.mapreduce.Job.(Job.java:145)
at org.apache.hadoop.mapreduce.Job.getInstance(Job.java:188)
at com.mstf.bigdata.hadoop.WordsCountAPP.main(WordsCountAPP.java:32)
[ERROR] method:org.apache.hadoop.util.Shell.getWinUtilsPath(Shell.java:425)
Failed to locate the winutils binary in the hadoop binary path
java.io.IOException: Could not locate executable null\bin\winutils.exe in the Hadoop binaries.
at org.apache.hadoop.util.Shell.getQualifiedBinPath(Shell.java:407)
at org.apache.hadoop.util.Shell.getWinUtilsPath(Shell.java:422)
at org.apache.hadoop.util.Shell.(Shell.java:415)
at org.apache.hadoop.util.StringUtils.(StringUtils.java:79)
at org.apache.hadoop.security.Groups.parseStaticMapping(Groups.java:168)
at org.apache.hadoop.security.Groups.(Groups.java:132)
at org.apache.hadoop.security.Groups.(Groups.java:100)
at org.apache.hadoop.security.Groups.getUserToGroupsMappingService(Groups.java:435)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:341)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:308)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:895)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:861)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:728)
at org.apache.hadoop.mapreduce.task.JobContextImpl.(JobContextImpl.java:72)
at org.apache.hadoop.mapreduce.Job.(Job.java:145)
at org.apache.hadoop.mapreduce.Job.getInstance(Job.java:188)
at com.mstf.bigdata.hadoop.WordsCountAPP.main(WordsCountAPP.java:32)
[DEBUG] method:org.apache.hadoop.security.Groups.(Groups.java:150)
Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000
[DEBUG] method:org.apache.hadoop.security.UserGroupInformation $H a d o o p L o g i n M o d u l e . l o g i n (U s e r G r o u p I n f o r m a t i o n . j a v a : 244) h a d o o p l o g i n [D E B U G] m e t h o d : o r g . a p a c h e . h a d o o p . s e c u r i t y . U s e r G r o u p I n f o r m a t i o n$ HadoopLoginModule.commit(UserGroupInformation.java:179)
hadoop login commit
[DEBUG] method:org.apache.hadoop.security.UserGroupInformation $H a d o o p L o g i n M o d u l e . c o m m i t (U s e r G r o u p I n f o r m a t i o n . j a v a : 215) U s i n g u s e r : " r o o t " w i t h n a m e r o o t [D E B U G] m e t h o d : o r g . a p a c h e . h a d o o p . s e c u r i t y . U s e r G r o u p I n f o r m a t i o n$ HadoopLoginModule.commit(UserGroupInformation.java:225)
User entry: “root”
[DEBUG] method:org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:940)
UGI loginUser:root (auth:SIMPLE)
[DEBUG] method:org.apache.htrace.core.Tracer $B u i l d e r . l o a d S a m p l e r s (T r a c e r . j a v a : 106) s a m p l e r . c l a s s e s =; l o a d e d n o s a m p l e r s [D E B U G] m e t h o d : o r g . a p a c h e . h t r a c e . c o r e . T r a c e r$ Builder.loadSpanReceivers(Tracer.java:128)
span.receiver.classes = ; loaded no span receivers
[DEBUG] method:org.apache.hadoop.hdfs.DFSClient $C o n f . < i n i t > (D F S C l i e n t . j a v a : 479) d f s . c l i e n t . u s e . l e g a c y . b l o c k r e a d e r . l o c a l = f a l s e [D E B U G] m e t h o d : o r g . a p a c h e . h a d o o p . h d f s . D F S C l i e n t$ Conf.(DFSClient.java:482)
dfs.client.read.shortcircuit = false
[DEBUG] method:org.apache.hadoop.hdfs.DFSClient $C o n f . < i n i t > (D F S C l i e n t . j a v a : 485) d f s . c l i e n t . d o m a i n . s o c k e t . d a t a . t r a f f i c = f a l s e [D E B U G] m e t h o d : o r g . a p a c h e . h a d o o p . h d f s . D F S C l i e n t$ Conf.(DFSClient.java:488)
dfs.domain.socket.path =
[DEBUG] method:org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:726)
Sets dfs.client.block.write.replace-datanode-on-failure.min-replication to 0
[DEBUG] method:org.apache.hadoop.io.retry.RetryUtils.getDefaultRetryPolicy(Retr