Eclipse+Maven构建Hadoop项目的方法步骤

<project xmlns= "http://maven.apache.org/pom/4.0.0" xmlns:xsi= "http://HdhCmsTestw3.org/2001/xmlschema-instance"

     xsi:schemalocation= "http://maven.apache.org/pom/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" >

     <modelversion> 4.0 . 0 </modelversion>

     <groupid>practice.hadoop</groupid>

     <artifactid>simple-examples</artifactid>

     <version> 0.0 . 1 -snapshot</version>

     <packaging>jar</packaging>

     <name>simple-examples</name>

     <url>http: //maven.apache.org</url>

     <properties>

         <project.build.sourceencoding>utf- 8 </project.build.sourceencoding>

     </properties>

     <dependencies>

         <dependency>

             <groupid>junit</groupid>

             <artifactid>junit</artifactid>

             <version> 4.12 </version>

             <scope>test</scope>

         </dependency>

         <!-- https: //mvnrepository测试数据/artifact/org.apache.hadoop/hadoop-common -->

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-common</artifactid>

             <version> 2.7 . 0 </version>

         </dependency>

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-hdfs</artifactid>

             <version> 2.7 . 0 </version>

         </dependency>

         <!-- https: //mvnrepository测试数据/artifact/org.apache.hadoop/hadoop-client -->

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-client</artifactid>

             <version> 2.7 . 0 </version>

         </dependency>

         <dependency>

             <groupid>org.apache.mrunit</groupid>

             <artifactid>mrunit</artifactid>

             <version> 1.1 . 0 </version>

             <classifier>hadoop2</classifier>

             <scope>test</scope>

         </dependency>

         <!-- https: //mvnrepository测试数据/artifact/org.apache.hadoop/hadoop-mapreduce-client-core -->

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-mapreduce-client-core</artifactid>

             <version> 2.7 . 0 </version>

         </dependency>

         <!-- https: //mvnrepository测试数据/artifact/org.apache.hadoop/hadoop-yarn-api -->

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-yarn-api</artifactid>

             <version> 2.7 . 0 </version>

         </dependency>

         <!-- https: //mvnrepository测试数据/artifact/org.apache.hadoop/hadoop-auth -->

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-auth</artifactid>

             <version> 2.7 . 0 </version>

         </dependency>

         <!-- https: //mvnrepository测试数据/artifact/org.apache.hadoop/hadoop-minicluster -->

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-minicluster</artifactid>

             <version> 2.7 . 0 </version>

             <scope>test</scope>

         </dependency>

         <!-- https: //mvnrepository测试数据/artifact/org.apache.hadoop/hadoop-mapreduce-client-jobclient -->

         <dependency>

             <groupid>org.apache.hadoop</groupid>

             <artifactid>hadoop-mapreduce-client-jobclient</artifactid>

             <version> 2.7 . 0 </version>

             <scope>provided</scope>

         </dependency>

     </dependencies>

</project>

import java.io.ioexception;

import java.util.stringtokenizer;

import org.apache.hadoop.conf.configuration;

import org.apache.hadoop.fs.path;

import org.apache.hadoop.io.intwritable;

import org.apache.hadoop.io.text;

import org.apache.hadoop.mapreduce.job;

import org.apache.hadoop.mapreduce.mapper;

import org.apache.hadoop.mapreduce.reducer;

import org.apache.hadoop.mapreduce.lib.input.fileinputformat;

import org.apache.hadoop.mapreduce.lib.output.fileoutputformat;

import org.apache.hadoop.util.genericoptionsparser;

public class wordcount {

     public static class tokenizermapper extends mapper<object, text, text, intwritable> {

         /**

          * longwritable, intwritable, text 均是 hadoop 中实现的用于封装 java

          * 数据类型的类，这些类实现了writablecomparable接口，

          * 都能够被串行化从而便于在分布式环境中进行数据交换，你可以将它们分别视为long,int,string 的替代品。

          */

         private final static intwritable one = new intwritable( 1 ); // 值为1

         private text word = new text();

         public void map(object key, text value, context context) throws ioexception, interruptedexception {

             stringtokenizer itr = new stringtokenizer(value.tostring()); // 对字符串进行切分

             while (itr.hasmoretokens()) {

                 word.set(itr.nexttoken());

                 context.write(word, one);

             }

         }

     }

     public static class intsumreducer extends reducer<text, intwritable, text, intwritable> {

         private intwritable result = new intwritable();

         public void reduce(text key, iterable<intwritable> values, context context)

                 throws ioexception, interruptedexception {

             int sum = 0 ;

             for (intwritable val : values) {

                 sum += val.get();

             }

             result.set(sum);

             context.write(key, result);

         }

     }

     public static void main(string[] args) throws exception {

         configuration conf = new configuration();

     conf.addresource( "classpath:/hadoop/core-site.xml" );

     conf.addresource( "classpath:/hadoop/hdfs-site.xml" );

     conf.addresource( "classpath:/hadoop/mapred-site.xml" );

//      string[] otherargs = new genericoptionsparser(conf, args).getremainingargs();

         string[] otherargs = { "/input" , "/output" };

         if (otherargs.length != 2 ) {

             system.err.println( "usage: wordcount <in> <out>" );

             system.exit( 2 );

         }

         job job = new job(conf, "word count" );

         job.setjarbyclass(wordcount. class );

         job.setmapperclass(tokenizermapper. class );

         job.setcombinerclass(intsumreducer. class );

         job.setreducerclass(intsumreducer. class );

         job.setoutputkeyclass(text. class );

         job.setoutputvalueclass(intwritable. class );

         fileinputformat.setinputdirrecursive(job, true );

         fileinputformat.addinputpath(job, new path(otherargs[ 0 ]));

         fileoutputformat.setoutputpath(job, new path(otherargs[ 1 ]));

         system.exit(job.waitforcompletion( true ) ? 0 : 1 );

     }

}