python 抓取搜狗指定公众号代码,Python代码库,德仔网

代码语言
代码分类
【Python】 python 抓取搜狗指定公众号
作者:首君 / 发布于2018/4/19/ 1094
#!/usr/bin/python
# -*- coding: utf-8 -*-
import urllib,urllib2,json,os,re,time,sys

def httpGet(url):
    header_dict = {
        'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
        "Referer": "http://weixin.sogou.com/weixin?query=%E8%87%AA%E7%94%B1%E5%85%89&_sug_type_=&s_from=input&_sug_=n&type=1&page=6&ie=utf8",
        "Cookie":"LSTMV=357%2C156; LCLKINT=3607",
        "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "Upgrade-Insecure-Requests": 1
        }
    try:
        req = urllib2.Request(url = url,headers=header_dict)
        res = urllib2.urlopen(req)
        result = res.read()
        return result
    except urllib2.URLError,e:
        if hasattr(e,"reason"):
            print "Failed to reach the server"
            print "The reason:",e.reason
        elif hasattr(e,"code"):
            print "The server couldn't fulfill the request"
            print "Error code:",e.code
            print "Return content:",e.read()
    else:
        pass  #其他异常的处理
    return '';

def writeFile(fileName,content):
    filePath = os.path.join(os.getcwd(),fileName)
    try:
        f = open(filePath,'a')
        f.write(content+'\n')
        f.close()
    except:
        print('error')
        pass;

def main(key):
    keys = urllib.quote((key).decode(sys.stdin.encoding).encode('utf8'))
    html = httpGet('http://weixin.sogou.com/weixin?type=1&s_from=input&query='+keys+'&ie=utf8&_sug_=y&_sug_type_=&w=01019900&sut=1268&sst0=1523857804947&lkt=0%2C0%2C0')

    nextPattern = re.compile(r'''<a id="sogou_next" href="([\S\s][^<>]*?)" class="np" uigs="page_next">''')
    nextUrl =  nextPattern.findall(html)
    if(nextUrl):
        nextUrl = 'http://weixin.sogou.com/weixin'+nextUrl[0];
    else:
        nextUrl = ''
    
    #href
    hrefPattern = re.compile(r'''<p class="tit">\n<a target="_blank" uigs=".*?" href="([\S\s][^<>]*?)">''');
    href = hrefPattern.findall(html)
    
    #微信名称
    titlePattern = re.compile(r'''<p class="tit">\n<a target="_blank" uigs=".*?" href=".*?">([\S\s]*?)</a>''');
    title = titlePattern.findall(html)

    #微信号
    namePattern = re.compile(r'''<label name="em_weixinhao">([\S\s]*?)</label>''');
    name = namePattern.findall(html)

    #功能介绍：
    funcPattern = re.compile(r'''<dt>功能介绍：</dt>\n<dd>([\S\s]*?)</dd>''');
    func = funcPattern.findall(html);

    content = '名称,微信号,链接,功能介绍\n'
    if(len(href)==len(title)==len(func)):
        for i in range(len(href)):
            content+='"'+removeHtml(title[i])+'","'+name[i]+'","'+'"'+href[i]+'","'+func[i]+'"\n'
        writeFile(key+'.csv',content.strip());
    else:
        print len(href),len(title),len(func)

    while(nextUrl!=''):
        nextUrl = page(key,nextUrl)

    print('end')

def page(key,nextUrl):
    html = httpGet(nextUrl)
    if(html==''):
        time.sleep(10)
        html = httpGet(nextUrl)
        if(html==''):
            print('error');

    nextPattern = re.compile(r'''<a id="sogou_next" href="([\S\s][^<>]*?)" class="np" uigs="page_next">''')
    nextUrl =  nextPattern.findall(html)
    if(nextUrl):
        nextUrl = 'http://weixin.sogou.com/weixin'+nextUrl[0];
    else:
        nextUrl = ''
    
    #href
    hrefPattern = re.compile(r'''<p class="tit">\n<a target="_blank" uigs=".*?" href="([\S\s][^<>]*?)">''');
    href = hrefPattern.findall(html)
    
    #微信名称
    titlePattern = re.compile(r'''<p class="tit">\n<a target="_blank" uigs=".*?" href=".*?">([\S\s]*?)</a>''');
    title = titlePattern.findall(html)

    #微信号
    namePattern = re.compile(r'''<label name="em_weixinhao">([\S\s]*?)</label>''');
    name = namePattern.findall(html)

    #功能介绍
    funcPattern = re.compile(r'''<dt>功能介绍：</dt>\n<dd>([\S\s]*?)</dd>''');
    func = funcPattern.findall(html);

    content = ''
    if(len(href)==len(title)==len(func)):
        for i in range(len(href)):
            content+='"'+removeHtml(title[i])+'","'+name[i]+'","'+'"'+href[i]+'","'+func[i]+'"\n'
        writeFile(key+'.csv',content.strip());
    else:
        print len(href),len(title),len(func)

    return nextUrl

def removeHtml(html):
    htmlPattern = re.compile(r'<[^>]+>',re.S)
    result = htmlPattern.sub('',html)
    return result;

if __name__=='__main__':
    main('自由人');
试试其它关键字
同语言下
可能有用的
首君贡献的其它代码(39)