idea maven 客户端运行wordcount

小提示

hadoop namenode -format 必须将将前面集群遗留的文件删除，不然datanode启动会失败

比需在resources下面给处hadoop集群的关键配置文件

wordcount 代码

package org.shangu.wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * @author : kean
 * @version V1.0
 * @Project: hadoop_spark_java
 * @Package org.shangu.wordcount
 * @Description: TODO
 * @date Date : 2019-05-02 15:58
 */

public class WordCountDriver {

    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);
        job.setJobName("wordcount");

        job.setJarByClass(WordCountDriver.class);

        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        boolean status = job.waitForCompletion(true);
        System.out.println(status);
    }
}

package org.shangu.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;
import java.util.StringTokenizer;

/**
 * @author : kean
 * @version V1.0
 * @Project: hadoop_spark_java
 * @Package org.shangu.wordcount
 * @Description: TODO
 * @date Date : 2019-05-02 15:23
 */

public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {

    private Text  text = new Text();

    private final static IntWritable value = new IntWritable(1);

    @Override
    public void map (LongWritable key, Text text, Context context ) throws IOException, InterruptedException {
        StringTokenizer itr = new StringTokenizer(text.toString());
        while (itr.hasMoreTokens()) {
            text.set(itr.nextToken());
            context.write(text, value);
        }
    }
}

package org.shangu.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 * @author : kean
 * @version V1.0
 * @Project: hadoop_spark_java
 * @Package org.shangu.wordcount
 * @Description: TODO
 * @date Date : 2019-05-02 15:39
 */

public class WordCountReducer extends Reducer<Text, IntWritable, Text,IntWritable> {

    private IntWritable value = new IntWritable();

    @Override
    public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum = 0;
        for (IntWritable val : values) {
            sum += val.get();
        }
        value.set(sum);
        context.write(key, value);
    }
}

执行环境变量

image.png

image.png

DHADOOP_USER_NAME=root取决于你的hdfs文件系统属于哪个用户


image.png

执行结果

D:\apps\jdk\bin\java.exe -DHADOOP_USER_NAME=root "-javaagent:D:\apps\IntelliJ IDEA 2019.1\lib\idea_rt.jar=14504:D:\apps\IntelliJ IDEA 2019.1\bin" -Dfile.encoding=UTF-8 -classpath D:\apps\jdk\jre\lib\charsets.jar;D:\apps\jdk\jre\lib\deploy.jar;D:\apps\jdk\jre\lib\ext\access-bridge-64.jar;D:\apps\jdk\jre\lib\ext\cldrdata.jar;D:\apps\jdk\jre\lib\ext\dnsns.jar;D:\apps\jdk\jre\lib\ext\jaccess.jar;D:\apps\jdk\jre\lib\ext\jfxrt.jar;D:\apps\jdk\jre\lib\ext\localedata.jar;D:\apps\jdk\jre\lib\ext\nashorn.jar;D:\apps\jdk\jre\lib\ext\sunec.jar;D:\apps\jdk\jre\lib\ext\sunjce_provider.jar;D:\apps\jdk\jre\lib\ext\sunmscapi.jar;D:\apps\jdk\jre\lib\ext\sunpkcs11.jar;D:\apps\jdk\jre\lib\ext\zipfs.jar;D:\apps\jdk\jre\lib\javaws.jar;D:\apps\jdk\jre\lib\jce.jar;D:\apps\jdk\jre\lib\jfr.jar;D:\apps\jdk\jre\lib\jfxswt.jar;D:\apps\jdk\jre\lib\jsse.jar;D:\apps\jdk\jre\lib\management-agent.jar;D:\apps\jdk\jre\lib\plugin.jar;D:\apps\jdk\jre\lib\resources.jar;D:\apps\jdk\jre\lib\rt.jar;D:\java_workspace\hadoop_spark_java\target\classes;D:\repository\org\apache\hadoop\hadoop-common\2.7.3\hadoop-common-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-annotations\2.7.3\hadoop-annotations-2.7.3.jar;D:\apps\jdk\lib\tools.jar;D:\repository\com\google\guava\guava\11.0.2\guava-11.0.2.jar;D:\repository\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;D:\repository\org\apache\commons\commons-math3\3.1.1\commons-math3-3.1.1.jar;D:\repository\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;D:\repository\commons-httpclient\commons-httpclient\3.1\commons-httpclient-3.1.jar;D:\repository\commons-codec\commons-codec\1.4\commons-codec-1.4.jar;D:\repository\commons-io\commons-io\2.4\commons-io-2.4.jar;D:\repository\commons-net\commons-net\3.1\commons-net-3.1.jar;D:\repository\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar;D:\repository\javax\servlet\servlet-api\2.5\servlet-api-2.5.jar;D:\repository\org\mortbay\jetty\jetty\6.1.26\jetty-6.1.26.jar;D:\repository\org\mortbay\jetty\jetty-util\6.1.26\jetty-util-6.1.26.jar;D:\repository\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;D:\repository\com\sun\jersey\jersey-core\1.9\jersey-core-1.9.jar;D:\repository\com\sun\jersey\jersey-json\1.9\jersey-json-1.9.jar;D:\repository\org\codehaus\jettison\jettison\1.1\jettison-1.1.jar;D:\repository\com\sun\xml\bind\jaxb-impl\2.2.3-1\jaxb-impl-2.2.3-1.jar;D:\repository\org\codehaus\jackson\jackson-jaxrs\1.8.3\jackson-jaxrs-1.8.3.jar;D:\repository\org\codehaus\jackson\jackson-xc\1.8.3\jackson-xc-1.8.3.jar;D:\repository\com\sun\jersey\jersey-server\1.9\jersey-server-1.9.jar;D:\repository\asm\asm\3.1\asm-3.1.jar;D:\repository\commons-logging\commons-logging\1.1.3\commons-logging-1.1.3.jar;D:\repository\net\java\dev\jets3t\jets3t\0.9.0\jets3t-0.9.0.jar;D:\repository\org\apache\httpcomponents\httpclient\4.1.2\httpclient-4.1.2.jar;D:\repository\org\apache\httpcomponents\httpcore\4.1.2\httpcore-4.1.2.jar;D:\repository\com\jamesmurty\utils\java-xmlbuilder\0.4\java-xmlbuilder-0.4.jar;D:\repository\commons-lang\commons-lang\2.6\commons-lang-2.6.jar;D:\repository\commons-configuration\commons-configuration\1.6\commons-configuration-1.6.jar;D:\repository\commons-digester\commons-digester\1.8\commons-digester-1.8.jar;D:\repository\commons-beanutils\commons-beanutils\1.7.0\commons-beanutils-1.7.0.jar;D:\repository\commons-beanutils\commons-beanutils-core\1.8.0\commons-beanutils-core-1.8.0.jar;D:\repository\org\slf4j\slf4j-api\1.7.10\slf4j-api-1.7.10.jar;D:\repository\org\slf4j\slf4j-log4j12\1.7.10\slf4j-log4j12-1.7.10.jar;D:\repository\org\codehaus\jackson\jackson-core-asl\1.9.13\jackson-core-asl-1.9.13.jar;D:\repository\org\codehaus\jackson\jackson-mapper-asl\1.9.13\jackson-mapper-asl-1.9.13.jar;D:\repository\org\apache\avro\avro\1.7.4\avro-1.7.4.jar;D:\repository\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;D:\repository\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;D:\repository\com\google\code\gson\gson\2.2.4\gson-2.2.4.jar;D:\repository\org\apache\hadoop\hadoop-auth\2.7.3\hadoop-auth-2.7.3.jar;D:\repository\org\apache\directory\server\apacheds-kerberos-codec\2.0.0-M15\apacheds-kerberos-codec-2.0.0-M15.jar;D:\repository\org\apache\directory\server\apacheds-i18n\2.0.0-M15\apacheds-i18n-2.0.0-M15.jar;D:\repository\org\apache\directory\api\api-asn1-api\1.0.0-M20\api-asn1-api-1.0.0-M20.jar;D:\repository\org\apache\directory\api\api-util\1.0.0-M20\api-util-1.0.0-M20.jar;D:\repository\org\apache\curator\curator-framework\2.7.1\curator-framework-2.7.1.jar;D:\repository\com\jcraft\jsch\0.1.42\jsch-0.1.42.jar;D:\repository\org\apache\curator\curator-client\2.7.1\curator-client-2.7.1.jar;D:\repository\org\apache\curator\curator-recipes\2.7.1\curator-recipes-2.7.1.jar;D:\repository\com\google\code\findbugs\jsr305\3.0.0\jsr305-3.0.0.jar;D:\repository\org\apache\htrace\htrace-core\3.1.0-incubating\htrace-core-3.1.0-incubating.jar;D:\repository\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;D:\repository\org\apache\commons\commons-compress\1.4.1\commons-compress-1.4.1.jar;D:\repository\org\tukaani\xz\1.0\xz-1.0.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-core\2.7.3\hadoop-mapreduce-client-core-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-yarn-common\2.7.3\hadoop-yarn-common-2.7.3.jar;D:\repository\javax\xml\bind\jaxb-api\2.2.2\jaxb-api-2.2.2.jar;D:\repository\javax\xml\stream\stax-api\1.0-2\stax-api-1.0-2.jar;D:\repository\javax\activation\activation\1.1\activation-1.1.jar;D:\repository\com\sun\jersey\jersey-client\1.9\jersey-client-1.9.jar;D:\repository\com\google\inject\guice\3.0\guice-3.0.jar;D:\repository\javax\inject\javax.inject\1\javax.inject-1.jar;D:\repository\aopalliance\aopalliance\1.0\aopalliance-1.0.jar;D:\repository\com\sun\jersey\contribs\jersey-guice\1.9\jersey-guice-1.9.jar;D:\repository\com\google\inject\extensions\guice-servlet\3.0\guice-servlet-3.0.jar;D:\repository\io\netty\netty\3.6.2.Final\netty-3.6.2.Final.jar;D:\repository\org\apache\spark\spark-core_2.11\2.2.1\spark-core_2.11-2.2.1.jar;D:\repository\org\apache\avro\avro-mapred\1.7.7\avro-mapred-1.7.7-hadoop2.jar;D:\repository\org\apache\avro\avro-ipc\1.7.7\avro-ipc-1.7.7.jar;D:\repository\org\apache\avro\avro-ipc\1.7.7\avro-ipc-1.7.7-tests.jar;D:\repository\com\twitter\chill_2.11\0.8.0\chill_2.11-0.8.0.jar;D:\repository\com\esotericsoftware\kryo-shaded\3.0.3\kryo-shaded-3.0.3.jar;D:\repository\com\esotericsoftware\minlog\1.3.0\minlog-1.3.0.jar;D:\repository\org\objenesis\objenesis\2.1\objenesis-2.1.jar;D:\repository\com\twitter\chill-java\0.8.0\chill-java-0.8.0.jar;D:\repository\org\apache\xbean\xbean-asm5-shaded\4.4\xbean-asm5-shaded-4.4.jar;D:\repository\org\apache\spark\spark-launcher_2.11\2.2.1\spark-launcher_2.11-2.2.1.jar;D:\repository\org\apache\spark\spark-network-common_2.11\2.2.1\spark-network-common_2.11-2.2.1.jar;D:\repository\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;D:\repository\com\fasterxml\jackson\core\jackson-annotations\2.6.5\jackson-annotations-2.6.5.jar;D:\repository\org\apache\spark\spark-network-shuffle_2.11\2.2.1\spark-network-shuffle_2.11-2.2.1.jar;D:\repository\org\apache\spark\spark-unsafe_2.11\2.2.1\spark-unsafe_2.11-2.2.1.jar;D:\repository\javax\servlet\javax.servlet-api\3.1.0\javax.servlet-api-3.1.0.jar;D:\repository\org\apache\commons\commons-lang3\3.5\commons-lang3-3.5.jar;D:\repository\org\slf4j\jul-to-slf4j\1.7.16\jul-to-slf4j-1.7.16.jar;D:\repository\org\slf4j\jcl-over-slf4j\1.7.16\jcl-over-slf4j-1.7.16.jar;D:\repository\com\ning\compress-lzf\1.0.3\compress-lzf-1.0.3.jar;D:\repository\org\xerial\snappy\snappy-java\1.1.2.6\snappy-java-1.1.2.6.jar;D:\repository\net\jpountz\lz4\lz4\1.3.0\lz4-1.3.0.jar;D:\repository\org\roaringbitmap\RoaringBitmap\0.5.11\RoaringBitmap-0.5.11.jar;D:\repository\org\scala-lang\scala-library\2.11.8\scala-library-2.11.8.jar;D:\repository\org\json4s\json4s-jackson_2.11\3.2.11\json4s-jackson_2.11-3.2.11.jar;D:\repository\org\json4s\json4s-core_2.11\3.2.11\json4s-core_2.11-3.2.11.jar;D:\repository\org\json4s\json4s-ast_2.11\3.2.11\json4s-ast_2.11-3.2.11.jar;D:\repository\org\scala-lang\scalap\2.11.0\scalap-2.11.0.jar;D:\repository\org\scala-lang\scala-compiler\2.11.0\scala-compiler-2.11.0.jar;D:\repository\org\scala-lang\modules\scala-xml_2.11\1.0.1\scala-xml_2.11-1.0.1.jar;D:\repository\org\scala-lang\modules\scala-parser-combinators_2.11\1.0.1\scala-parser-combinators_2.11-1.0.1.jar;D:\repository\org\glassfish\jersey\core\jersey-client\2.22.2\jersey-client-2.22.2.jar;D:\repository\javax\ws\rs\javax.ws.rs-api\2.0.1\javax.ws.rs-api-2.0.1.jar;D:\repository\org\glassfish\hk2\hk2-api\2.4.0-b34\hk2-api-2.4.0-b34.jar;D:\repository\org\glassfish\hk2\hk2-utils\2.4.0-b34\hk2-utils-2.4.0-b34.jar;D:\repository\org\glassfish\hk2\external\aopalliance-repackaged\2.4.0-b34\aopalliance-repackaged-2.4.0-b34.jar;D:\repository\org\glassfish\hk2\external\javax.inject\2.4.0-b34\javax.inject-2.4.0-b34.jar;D:\repository\org\glassfish\hk2\hk2-locator\2.4.0-b34\hk2-locator-2.4.0-b34.jar;D:\repository\org\javassist\javassist\3.18.1-GA\javassist-3.18.1-GA.jar;D:\repository\org\glassfish\jersey\core\jersey-common\2.22.2\jersey-common-2.22.2.jar;D:\repository\javax\annotation\javax.annotation-api\1.2\javax.annotation-api-1.2.jar;D:\repository\org\glassfish\jersey\bundles\repackaged\jersey-guava\2.22.2\jersey-guava-2.22.2.jar;D:\repository\org\glassfish\hk2\osgi-resource-locator\1.0.1\osgi-resource-locator-1.0.1.jar;D:\repository\org\glassfish\jersey\core\jersey-server\2.22.2\jersey-server-2.22.2.jar;D:\repository\org\glassfish\jersey\media\jersey-media-jaxb\2.22.2\jersey-media-jaxb-2.22.2.jar;D:\repository\javax\validation\validation-api\1.1.0.Final\validation-api-1.1.0.Final.jar;D:\repository\org\glassfish\jersey\containers\jersey-container-servlet\2.22.2\jersey-container-servlet-2.22.2.jar;D:\repository\org\glassfish\jersey\containers\jersey-container-servlet-core\2.22.2\jersey-container-servlet-core-2.22.2.jar;D:\repository\io\netty\netty-all\4.0.43.Final\netty-all-4.0.43.Final.jar;D:\repository\com\clearspring\analytics\stream\2.7.0\stream-2.7.0.jar;D:\repository\io\dropwizard\metrics\metrics-core\3.1.2\metrics-core-3.1.2.jar;D:\repository\io\dropwizard\metrics\metrics-jvm\3.1.2\metrics-jvm-3.1.2.jar;D:\repository\io\dropwizard\metrics\metrics-json\3.1.2\metrics-json-3.1.2.jar;D:\repository\io\dropwizard\metrics\metrics-graphite\3.1.2\metrics-graphite-3.1.2.jar;D:\repository\com\fasterxml\jackson\core\jackson-databind\2.6.5\jackson-databind-2.6.5.jar;D:\repository\com\fasterxml\jackson\core\jackson-core\2.6.5\jackson-core-2.6.5.jar;D:\repository\com\fasterxml\jackson\module\jackson-module-scala_2.11\2.6.5\jackson-module-scala_2.11-2.6.5.jar;D:\repository\org\scala-lang\scala-reflect\2.11.7\scala-reflect-2.11.7.jar;D:\repository\com\fasterxml\jackson\module\jackson-module-paranamer\2.6.5\jackson-module-paranamer-2.6.5.jar;D:\repository\org\apache\ivy\ivy\2.4.0\ivy-2.4.0.jar;D:\repository\oro\oro\2.0.8\oro-2.0.8.jar;D:\repository\net\razorvine\pyrolite\4.13\pyrolite-4.13.jar;D:\repository\net\sf\py4j\py4j\0.10.4\py4j-0.10.4.jar;D:\repository\org\apache\spark\spark-tags_2.11\2.2.1\spark-tags_2.11-2.2.1.jar;D:\repository\org\apache\commons\commons-crypto\1.0.0\commons-crypto-1.0.0.jar;D:\repository\org\spark-project\spark\unused\1.0.0\unused-1.0.0.jar;D:\repository\org\apache\commons\commons-math\2.2\commons-math-2.2.jar;D:\repository\org\apache\hadoop\hadoop-client\2.7.3\hadoop-client-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-hdfs\2.7.3\hadoop-hdfs-2.7.3.jar;D:\repository\xerces\xercesImpl\2.9.1\xercesImpl-2.9.1.jar;D:\repository\xml-apis\xml-apis\1.3.04\xml-apis-1.3.04.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-app\2.7.3\hadoop-mapreduce-client-app-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-common\2.7.3\hadoop-mapreduce-client-common-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-yarn-client\2.7.3\hadoop-yarn-client-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-yarn-server-common\2.7.3\hadoop-yarn-server-common-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-shuffle\2.7.3\hadoop-mapreduce-client-shuffle-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-yarn-api\2.7.3\hadoop-yarn-api-2.7.3.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-jobclient\2.7.3\hadoop-mapreduce-client-jobclient-2.7.3.jar;D:\repository\log4j\log4j\1.2.17\log4j-1.2.17.jar org.shangu.wordcount.WordCountDriver /wordcount/input /wordcount/output
2019-05-02 17:17:21  [main] [INFO]  - session.id is deprecated. Instead, use dfs.metrics.session-id
2019-05-02 17:17:21  [main] [INFO]  - Initializing JVM Metrics with processName=JobTracker, sessionId=
2019-05-02 17:17:32  [main] [WARN]  - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
2019-05-02 17:17:32  [main] [WARN]  - No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
2019-05-02 17:17:32  [main] [INFO]  - Total input paths to process : 1
2019-05-02 17:17:32  [main] [INFO]  - number of splits:1
2019-05-02 17:17:32  [main] [INFO]  - Submitting tokens for job: job_local1926305669_0001
2019-05-02 17:17:32  [main] [INFO]  - The url to track the job: http://localhost:8080/
2019-05-02 17:17:32  [main] [INFO]  - Running job: job_local1926305669_0001
2019-05-02 17:17:32  [Thread-4] [INFO]  - OutputCommitter set in config null
2019-05-02 17:17:32  [Thread-4] [INFO]  - File Output Committer Algorithm version is 1
2019-05-02 17:17:32  [Thread-4] [INFO]  - OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
2019-05-02 17:17:32  [Thread-4] [INFO]  - Waiting for map tasks
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - Starting task: attempt_local1926305669_0001_m_000000_0
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - File Output Committer Algorithm version is 1
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - ProcfsBasedProcessTree currently is supported only on Linux.
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  -  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@68c046f0
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - Processing split: hdfs://172.16.21.220:9000/wordcount/input/xiyouji.txt:0+2171334
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - (EQUATOR) 0 kvi 26214396(104857584)
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - mapreduce.task.io.sort.mb: 100
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - soft limit at 83886080
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - bufstart = 0; bufvoid = 104857600
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - kvstart = 26214396; length = 6553600
2019-05-02 17:17:32  [LocalJobRunner Map Task Executor #0] [INFO]  - Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
2019-05-02 17:17:33  [main] [INFO]  - Job job_local1926305669_0001 running in uber mode : false
2019-05-02 17:17:33  [main] [INFO]  -  map 0% reduce 0%
2019-05-02 17:17:33  [LocalJobRunner Map Task Executor #0] [INFO]  - Found UTF-8 BOM and skipped it
2019-05-02 17:17:33  [LocalJobRunner Map Task Executor #0] [INFO]  - 
2019-05-02 17:17:33  [LocalJobRunner Map Task Executor #0] [INFO]  - Starting flush of map output
2019-05-02 17:17:33  [LocalJobRunner Map Task Executor #0] [INFO]  - Spilling map output
2019-05-02 17:17:33  [LocalJobRunner Map Task Executor #0] [INFO]  - bufstart = 0; bufend = 2169960; bufvoid = 104857600
2019-05-02 17:17:33  [LocalJobRunner Map Task Executor #0] [INFO]  - kvstart = 26214396(104857584); kvend = 26200740(104802960); length = 13657/6553600
2019-05-02 17:17:33  [LocalJobRunner Map Task Executor #0] [INFO]  - Finished spill 0
2019-05-02 17:17:34  [LocalJobRunner Map Task Executor #0] [INFO]  - Task:attempt_local1926305669_0001_m_000000_0 is done. And is in the process of committing
2019-05-02 17:17:34  [LocalJobRunner Map Task Executor #0] [INFO]  - map
2019-05-02 17:17:34  [LocalJobRunner Map Task Executor #0] [INFO]  - Task 'attempt_local1926305669_0001_m_000000_0' done.
2019-05-02 17:17:34  [LocalJobRunner Map Task Executor #0] [INFO]  - Finishing task: attempt_local1926305669_0001_m_000000_0
2019-05-02 17:17:34  [Thread-4] [INFO]  - map task executor complete.
2019-05-02 17:17:34  [Thread-4] [INFO]  - Waiting for reduce tasks
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Starting task: attempt_local1926305669_0001_r_000000_0
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - File Output Committer Algorithm version is 1
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - ProcfsBasedProcessTree currently is supported only on Linux.
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  -  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@1e712dff
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@3cbb9b3e
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - MergerManager: memoryLimit=2650485504, maxSingleShuffleLimit=662621376, mergeThreshold=1749320448, ioSortFactor=10, memToMemMergeOutputsThreshold=10
2019-05-02 17:17:34  [EventFetcher for fetching Map Completion Events] [INFO]  - attempt_local1926305669_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
2019-05-02 17:17:34  [localfetcher#1] [INFO]  - localfetcher#1 about to shuffle output of map attempt_local1926305669_0001_m_000000_0 decomp: 2181848 len: 2181852 to MEMORY
2019-05-02 17:17:34  [localfetcher#1] [INFO]  - Read 2181848 bytes from map-output for attempt_local1926305669_0001_m_000000_0
2019-05-02 17:17:34  [localfetcher#1] [INFO]  - closeInMemoryFile -> map-output of size: 2181848, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->2181848
2019-05-02 17:17:34  [EventFetcher for fetching Map Completion Events] [INFO]  - EventFetcher is interrupted.. Returning
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - 1 / 1 copied.
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Merging 1 sorted segments
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Down to the last merge-pass, with 1 segments left of total size: 2181830 bytes
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Merged 1 segments, 2181848 bytes to disk to satisfy reduce memory limit
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Merging 1 files, 2181852 bytes from disk
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Merging 0 segments, 0 bytes from memory into reduce
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Merging 1 sorted segments
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Down to the last merge-pass, with 1 segments left of total size: 2181830 bytes
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - 1 / 1 copied.
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
2019-05-02 17:17:34  [main] [INFO]  -  map 100% reduce 0%
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Task:attempt_local1926305669_0001_r_000000_0 is done. And is in the process of committing
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - 1 / 1 copied.
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Task attempt_local1926305669_0001_r_000000_0 is allowed to commit now
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Saved output of task 'attempt_local1926305669_0001_r_000000_0' to hdfs://172.16.21.220:9000/wordcount/output/_temporary/0/task_local1926305669_0001_r_000000
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - reduce > reduce
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Task 'attempt_local1926305669_0001_r_000000_0' done.
2019-05-02 17:17:34  [pool-6-thread-1] [INFO]  - Finishing task: attempt_local1926305669_0001_r_000000_0
2019-05-02 17:17:34  [Thread-4] [INFO]  - reduce task executor complete.
2019-05-02 17:17:35  [main] [INFO]  -  map 100% reduce 100%
2019-05-02 17:17:35  [main] [INFO]  - Job job_local1926305669_0001 completed successfully
2019-05-02 17:17:35  [main] [INFO]  - Counters: 35
    File System Counters
        FILE: Number of bytes read=4364098
        FILE: Number of bytes written=7117898
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
        HDFS: Number of bytes read=4342668
        HDFS: Number of bytes written=2156551
        HDFS: Number of read operations=13
        HDFS: Number of large read operations=0
        HDFS: Number of write operations=4
    Map-Reduce Framework
        Map input records=6823
        Map output records=3415
        Map output bytes=2169960
        Map output materialized bytes=2181852
        Input split bytes=118
        Combine input records=0
        Combine output records=0
        Reduce input groups=3311
        Reduce shuffle bytes=2181852
        Reduce input records=3415
        Reduce output records=3311
        Spilled Records=6830
        Shuffled Maps =1
        Failed Shuffles=0
        Merged Map outputs=1
        GC time elapsed (ms)=9
        Total committed heap usage (bytes)=508035072
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=2171334
    File Output Format Counters 
        Bytes Written=2156551
true

Process finished with exit code 0
©著作权归作者所有,转载或内容合作请联系作者
  • 人面猴
    序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
    沈念sama阅读 212,185评论 6赞 493
  • 死咒
    序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
    沈念sama阅读 90,445评论 3赞 385
  • 救了他两次的神仙让他今天三更去死
    文/潘晓璐 我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
    开封第一讲书人阅读 157,684评论 0赞 348
  • 道士缉凶录：失踪的卖姜人
    文/不坏的土叔 我叫张陵，是天一观的道长。 经常有香客问我，道长，这世上最难降的妖魔是什么？ 我笑而不...
    开封第一讲书人阅读 56,564评论 1赞 284
  • 港岛之恋（遗憾婚礼）
    正文 为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
    茶点故事阅读 65,681评论 6赞 386
  • 恶毒庶女顶嫁案：这布局不是一般人想出来的
    文/花漫 我一把揭开白布。 她就那样静静地躺着，像睡着了一般。 火红的嫁衣衬着肌肤如雪。 梳的纹丝不乱的头发上，一...
    开封第一讲书人阅读 49,874评论 1赞 290
  • 城市分裂传说
    那天，我揣着相机与录音，去河边找鬼。 笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。 我是一名探鬼主播，决...
    沈念sama阅读 39,025评论 3赞 408
  • 双鸳鸯连环套：你想象不到人心有多黑
    文/苍兰香墨 我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
    开封第一讲书人阅读 37,761评论 0赞 268
  • 万荣杀人案实录
    序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
    沈念sama阅读 44,217评论 1赞 303
  • 护林员之死
    正文 独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋 以下内容为张勋视角 年9月15日...
    茶点故事阅读 36,545评论 2赞 327
  • 白月光启示录
    正文 我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。 大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
    茶点故事阅读 38,694评论 1赞 341
  • 活死人
    序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
    沈念sama阅读 34,351评论 4赞 332
  • 日本核电站爆炸内幕
    正文 年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
    茶点故事阅读 39,988评论 3赞 315
  • 男人毒药：我在死后第九天来索命
    文/蒙蒙 一、第九天 我趴在偏房一处隐蔽的房顶上张望。 院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
    开封第一讲书人阅读 30,778评论 0赞 21
  • 一桩弑父案，背后竟有这般阴谋
    文/苍兰香墨 我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。 一阵脚步声响...
    开封第一讲书人阅读 32,007评论 1赞 266
  • 情欲美人皮
    我被黑心中介骗来泰国打工， 没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。 一个月前我还...
    沈念sama阅读 46,427评论 2赞 360
  • 代替公主和亲
    正文 我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。 传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
    茶点故事阅读 43,580评论 2赞 349

推荐阅读更多精彩内容