java爬取并下载酷狗TOP500歌曲的方法

var datafromsmarty = [{ "hash" : "667939c6e784265d541deee65ae4f2f8" , "timelength" : "237051" , "audio_name" : "u767du5c0fu767d - u6700u7f8eu5a5au793c" , "author_name" : "u767du5c0fu767d" , "song_name" : "u6700u7f8eu5a5au793c" , "album_id" : 0 }], //当前页面歌曲信息

       playtype = "search_single" ; //当前播放

   </script>

package com.bing.http;

import java.io.bufferedreader;

import java.io.ioexception;

import java.io.inputstream;

import java.io.inputstreamreader;

import java.security.nosuchalgorithmexception;

import java.security.cert.certificateexception;

import java.security.cert.x509certificate;

import javax.net.ssl.sslcontext;

import javax.net.ssl.trustmanager;

import javax.net.ssl.x509trustmanager;

import org.apache.commons.logging.log;

import org.apache.commons.logging.logfactory;

import org.apache.http.httpentity;

import org.apache.http.client.clientprotocolexception;

import org.apache.http.client.httpclient;

import org.apache.http.client.responsehandler;

import org.apache.http.client.config.requestconfig;

import org.apache.http.client.methods.closeablehttpresponse;

import org.apache.http.client.methods.httpget;

import org.apache.http.conn.clientconnectionmanager;

import org.apache.http.conn.scheme.scheme;

import org.apache.http.conn.scheme.schemeregistry;

import org.apache.http.conn.ssl.sslsocketfactory;

import org.apache.http.impl.client.basicresponsehandler;

import org.apache.http.impl.client.closeablehttpclient;

import org.apache.http.impl.client.defaulthttpclient;

import org.apache.http.impl.client.httpclients;

import org.apache.http.impl.conn.basichttpclientconnectionmanager;

import org.apache.http.params.httpparams;

/**

  * @说明：

  * @author: gaoll

  * @createtime:2014-11-13

  * @modifytime:2014-11-13

  */

public class httpgetconnect {

     /**

      * 获取html内容

      * @param url

      * @param charsetname utf-8、gb2312

      * @return

      * @throws ioexception

      */

     public static string connect(string url,string charsetname) throws ioexception{

         basichttpclientconnectionmanager connmanager = new basichttpclientconnectionmanager();

         closeablehttpclient httpclient = httpclients.custom()

           .setconnectionmanager(connmanager)

           .build();

         string content = "" ;

         try {

             httpget httpget = new httpget(url);

             requestconfig requestconfig = requestconfig.custom()

             .setsockettimeout( 5000 )

             .setconnecttimeout( 50000 )

             .setconnectionrequesttimeout( 50000 )

             .build();

         httpget.setconfig(requestconfig);

         httpget.setheader( "accept" , "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" );

         httpget.setheader( "accept-encoding" , "gzip,deflate,sdch" );

         httpget.setheader( "accept-language" , "zh-cn,zh;q=0.8" );

         httpget.setheader( "connection" , "keep-alive" );

         httpget.setheader( "upgrade-insecure-requests" , "1" );

         httpget.setheader( "user-agent" , "mozilla/5.0 (windows nt 6.1; wow64) applewebkit/537.36 (khtml, like gecko) chrome/45.0.2454.101 safari/537.36" );

         //httpget.setheader("hosts", "www.oschina.net");

         httpget.setheader( "cache-control" , "max-age=0" );

             closeablehttpresponse response = httpclient.execute(httpget);

             int status = response.getstatusline().getstatuscode();

       if (status >= 200 && status < 300 ) {

         httpentity entity = response.getentity();

         inputstream instream = entity.getcontent();

         bufferedreader br = new bufferedreader( new inputstreamreader(instream,charsetname));

                 stringbuffer sbf = new stringbuffer();

                 string line = null ;

                 while ((line = br.readline()) != null ){

                     sbf.append(line + " " );

                 }

                 br.close();

                 content = sbf.tostring();

       } else {

         content = "" ;

       }

         } catch (exception e){

             e.printstacktrace();

         } finally {

             httpclient.close();

         }

         //log.info("content is " + content);

         return content;

     }

     private static log log = logfactory.getlog(httpgetconnect. class );

}

package com.bing.html;

import java.io.ioexception;

import java.util.arraylist;

import java.util.list;

import org.apache.commons.logging.log;

import org.apache.commons.logging.logfactory;

import org.jsoup.jsoup;

import org.jsoup.nodes.document;

import org.jsoup.nodes.element;

import org.jsoup.select.elements;

import com.bing.http.httpgetconnect;

/**

  * @说明：

  * @author: gaoll

  * @createtime:2014-11-13

  * @modifytime:2014-11-13

  */

public class htmlmanage {

     public document manage(string html){

         document doc = jsoup.parse(html);

         return doc;

     }

     public document managedirect(string url) throws ioexception{

         document doc = jsoup.connect( url ).get();

         return doc;

     }

     public list<string> managehtmltag(document doc,string tag ){

         list<string> list = new arraylist<string>();

         elements elements = doc.getelementsbytag(tag);

         for ( int i = 0 ; i < elements.size() ; i++){

             string str = elements.get(i).html();

             list.add(str);

         }

         return list;

     }

     public list<string> managehtmlclass(document doc,string clas ){

         list<string> list = new arraylist<string>();

         elements elements = doc.getelementsbyclass(clas);

         for ( int i = 0 ; i < elements.size() ; i++){

             string str = elements.get(i).html();

             list.add(str);

         }

         return list;

     }

     public list<string> managehtmlkey(document doc,string key,string value ){

         list<string> list = new arraylist<string>();

         elements elements = doc.getelementsbyattributevalue(key, value);

         for ( int i = 0 ; i < elements.size() ; i++){

             string str = elements.get(i).html();

             list.add(str);

         }

         return list;

     }

     private static log log = logfactory.getlog(htmlmanage. class );

}

package com.bing.download;

import java.io.bufferedinputstream;

import java.io.bufferedoutputstream;

import java.io.file;

import java.io.fileoutputstream;

import org.apache.commons.logging.log;

import org.apache.commons.logging.logfactory;

import org.apache.http.client.config.requestconfig;

import org.apache.http.client.methods.closeablehttpresponse;

import org.apache.http.client.methods.httpget;

import org.apache.http.impl.client.closeablehttpclient;

import org.apache.http.impl.client.httpclients;

/**

  * @说明：

  * @author: gaoll

  * @createtime:2014-11-20

  * @modifytime:2014-11-20

  */

public class filedownload {

     /**

      * 文件下载

      * @param url 链接地址

      * @param path 要保存的路径及文件名

      * @return

      */

     public static boolean download(string url,string path){

         boolean flag = false ;

         closeablehttpclient httpclient = httpclients.createdefault();

         requestconfig requestconfig = requestconfig.custom().setsockettimeout( 2000 )

                 .setconnecttimeout( 2000 ).build();

         httpget get = new httpget(url);

         get.setconfig(requestconfig);

         bufferedinputstream in = null ;

         bufferedoutputstream out = null ;

         try {

             for ( int i= 0 ;i< 3 ;i++){

                 closeablehttpresponse result = httpclient.execute(get);

                 system.out.println(result.getstatusline());

                 if (result.getstatusline().getstatuscode() == 200 ){

                     in = new bufferedinputstream(result.getentity().getcontent());

                     file file = new file(path);

                     out = new bufferedoutputstream( new fileoutputstream(file));

                     byte [] buffer = new byte [ 1024 ];

                     int len = - 1 ;

                     while ((len = in.read(buffer, 0 , 1024 )) > - 1 ){

                         out.write(buffer, 0 ,len);

                     }

                     flag = true ;

                     break ;

                 } else if (result.getstatusline().getstatuscode() == 500 ){

                     continue ;

                 }

             }

         } catch (exception e){

             e.printstacktrace();

             flag = false ;

         } finally {

             get.releaseconnection();

             try {

                 if (in != null ){

                     in.close();

                 }

                 if (out != null ){

                     out.close();

                 }

             } catch (exception e){

                 e.printstacktrace();

                 flag = false ;

             }

         }

         return flag;

     }

     private static log log = logfactory.getlog(filedownload. class );

}