Java基于WebMagic爬取某豆瓣电影评论的实现

package us.codecraft.webmagic.processor.example;

import us.codecraft.webmagic.page;

import us.codecraft.webmagic.resultitems;

import us.codecraft.webmagic.site;

import us.codecraft.webmagic.spider;

import us.codecraft.webmagic.processor.pageprocessor;

import java.util.list;

import java.util.map;

/**

  * a simple pageprocessor.

  * 爬取豆瓣某电影的评论  爬取地址：https://movie.douban.com/subject/35096844/reviews?start=0

  *

  * @author code4crafter@gmail.com <br>

  * @since 0.1.0

  */

public class doubanpageprocessor implements pageprocessor {

   private site site;

   public doubanpageprocessor(string urlpattern) {

     this .site = site.me().setretrytimes( 3 ).setsleeptime( 300 ); // 设置站点重试次数3 间隔300ms

   }

   @override

   public void process(page page) {

     page.putfield( "title" , page.gethtml().xpath( "//title/text()" )); //爬取网页标题

//    page.putfield("html", page.gethtml().tostring()); //爬取整个页面的html

     page.putfield( "titlelist" , page.gethtml().css( "div.short-content" , "text" ).all()); // 我们要爬取的核心信息内容，获取方式与css选择器用法一样

//    page.putfield("content", page.gethtml().smartcontent());

   }

   @override

   public site getsite() {

     //settings

     return site;

   }

   public static void main(string[] args) {

     spider spider = spider.create( new doubanpageprocessor( "https://movie\\.douban\\.com\\d+" ));

     resultitems resultitems = spider.<resultitems>get( "https://movie.douban.com/subject/35096844/reviews?start=0" );// 爬取并获得爬取结果

     map<string, object> map = resultitems.getall();

     for (map.entry entry : map.entryset()) {

       system.out.println(entry.getkey() + " : " + entry.getvalue()); //打印爬取的所有内容

     }

     list<string> shortlist = (list<string>) map.get( "titlelist" );

     system.out.println( "=====================分隔线===================\n短评如下：" );

     for ( int i = 0 ; i < shortlist.size(); i++) {

       system.out.println(i + "、" + shortlist.get(i).trim()); // 打印爬取的评论内容

     }

     spider.close();

   }

}