import html import requests from pyquery import PyQuery as pq BASE_URL = 'https://www.cheese.com' class CheeseComError(Exception): pass class CheeseCom(object): def query(self, q): try: r = requests.get(BASE_URL, params=dict(q=q)) d = pq(r.content) if not r.history: url = d('div[class~="cheese-item"]').eq(0).find('a').eq(0).attr('href') r = requests.get(BASE_URL + url) d = pq(r.content) summary = [] for p in d('ul[class~="summary-points"]').find('p').items(): summary.append(html.unescape(p.html(method='text'))) return dict( url=d('meta[name="twitter:url"]').attr('content'), name=html.unescape(d('meta[name="twitter:title"]').attr('content')), description=html.unescape(d('meta[name="twitter:description"]').attr('content')), image=d('meta[name="twitter:image"]').attr('content'), summary=summary) except Exception as e: raise CheeseComError(str(e))