#!/usr/bin/python import pycurl,StringIO def crawl(name): crl = pycurl.Curl() crl.setopt(pycurl.VERBOSE,0) # No verbose info print crl.setopt(pycurl.NOSIGNAL, 1) # avoid memory dump error crl.setopt(pycurl.FOLLOWLOCATION, 1) crl.setopt(pycurl.MAXREDIRS, 5) crl.fp = StringIO.StringIO() crl.setopt(pycurl.URL, url) crl.setopt(crl.WRITEFUNCTION, crl.fp.write) crl.perform() page = crl.fp.getvalue()