=== modified file 'ubuntu_sso/account.py' --- ubuntu_sso/account.py 2011-11-23 19:49:20 +0000 +++ ubuntu_sso/account.py 2011-12-15 18:31:17 +0000 @@ -20,7 +20,6 @@ import os import re -import urllib2 # Unable to import 'lazr.restfulclient.*' # pylint: disable=F0401 @@ -32,7 +31,7 @@ from oauth import oauth from ubuntu_sso.logger import setup_logging -from ubuntu_sso.utils import timestamp_checker +from ubuntu_sso.utils import curllib, timestamp_checker logger = setup_logging("ubuntu_sso.account") @@ -140,7 +139,7 @@ # download captcha and save to 'filename' logger.debug('generate_captcha: server answered: %r', captcha) try: - res = urllib2.urlopen(captcha['image_url']) + res = curllib.urlopen(captcha['image_url']) with open(filename, 'wb') as f: f.write(res.read()) except: === modified file 'ubuntu_sso/credentials.py' --- ubuntu_sso/credentials.py 2011-11-23 19:49:20 +0000 +++ ubuntu_sso/credentials.py 2011-12-15 18:32:18 +0000 @@ -39,7 +39,6 @@ import sys import traceback -import urllib2 from functools import wraps @@ -49,6 +48,7 @@ from ubuntu_sso import NO_OP, utils from ubuntu_sso.keyring import Keyring from ubuntu_sso.logger import setup_logging +from ubuntu_sso.utils import curllib logger = setup_logging('ubuntu_sso.credentials') @@ -245,13 +245,13 @@ parameters=parameters) oauth_req.sign_request(oauth.OAuthSignatureMethod_HMAC_SHA1(), consumer, token) - request = urllib2.Request(url, headers=oauth_req.to_header()) - logger.debug('Opening the url "%s" with urllib2.urlopen.', + request = curllib.Request(url, headers=oauth_req.to_header()) + logger.debug('Opening the url "%s" with curllib.urlopen.', request.get_full_url()) # This code is blocking, we should change this. # I've tried with deferToThread an twisted.web.client.getPage # but the returned deferred will never be fired (nataliabidart). - response = urllib2.urlopen(request) + response = curllib.urlopen(request) logger.debug('Url opened. Response: %s.', response.code) returnValue(response.code) === modified file 'ubuntu_sso/tests/test_credentials.py' --- ubuntu_sso/tests/test_credentials.py 2011-11-23 19:49:20 +0000 +++ ubuntu_sso/tests/test_credentials.py 2011-12-15 18:33:11 +0000 @@ -280,7 +280,7 @@ self.patch(credentials.Keyring, 'get_credentials', lambda kr, app: defer.succeed(TOKEN)) error = 'Bla' - self.patch(credentials.urllib2, 'urlopen', + self.patch(credentials.curllib, 'urlopen', lambda *a, **kw: self.fail(error)) self._cred_cleared = False self.patch(self.obj, 'clear_credentials', @@ -359,14 +359,14 @@ code=200) return response - self.patch(credentials.urllib2, 'urlopen', faked_urlopen) + self.patch(credentials.curllib, 'urlopen', faked_urlopen) self.patch(credentials.utils.timestamp_checker, "get_faithful_time", time.time) @inlineCallbacks def test_ping_url_if_url_is_none(self): """self.ping_url is opened.""" - self.patch(credentials.urllib2, 'urlopen', self.fail) + self.patch(credentials.curllib, 'urlopen', self.fail) self.obj.ping_url = None yield self.obj._ping_url(app_name=APP_NAME, email=EMAIL, credentials=TOKEN) @@ -378,7 +378,7 @@ yield self.obj._ping_url(app_name=APP_NAME, email=EMAIL, credentials=TOKEN) - self.assertIsInstance(self._request, credentials.urllib2.Request) + self.assertIsInstance(self._request, credentials.curllib.Request) self.assertEqual(self._request.get_full_url(), self.obj.ping_url + EMAIL) @@ -402,7 +402,7 @@ def test_ping_url_error(self): """Exception is handled if ping fails.""" error = 'Blu' - self.patch(credentials.urllib2, 'urlopen', lambda r: self.fail(error)) + self.patch(credentials.curllib, 'urlopen', lambda r: self.fail(error)) yield self.obj._ping_url(APP_NAME, EMAIL, TOKEN) === added file 'ubuntu_sso/utils/curllib.py' --- ubuntu_sso/utils/curllib.py 1970-01-01 00:00:00 +0000 +++ ubuntu_sso/utils/curllib.py 2011-12-15 18:31:17 +0000 @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- + +# Author: Alejandro J. Cura +# +# Copyright 2011 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 3, as published +# by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +"""Web client based on pycurl, with an API similar to urllib2.""" + +import httplib + +from email import parser +from StringIO import StringIO + +import pycurl + + +class HTTPError(Exception): + """Error that happens while doing a web request.""" + + def __init__(self, url, code, message, headers=None, fp=None): + """Initialize this instance.""" + super(HTTPError, self).__init__() + self.code = code + self.message = message + if fp: + self.fp = fp + else: + self.fp = StringIO(message) + + def read(self): + """Return the error message.""" + return self.fp.read() + + def __str__(self): + return "<%s: %s>" % (self.code, self.message) + + +class UnauthorizedError(HTTPError): + """An HTTP error when there's an auth problem.""" + + +class Request(object): + """An HTTP request object.""" + + def __init__(self, url, data=None, headers=None): + """Initialize this instance.""" + super(Request, self).__init__() + self.url = url + self.data = data + self.headers = headers + + def get_full_url(self): + """Return the url.""" + return self.url + + def get_data(self): + """Return the data.""" + return self.data + + +class Response(StringIO): + """An HTTP response object.""" + + code = -1 + headers = None + + def __init__(self, url): + """Initialize this instance.""" + StringIO.__init__(self) + self.url = url + + def finish(self, code, headers): + """Finish a response and rewind it.""" + self.code = code + self.headers = headers + self.seek(0) + + +class HeaderParser(parser.FeedParser): + """A parser for the HTTP headers, based on the stdlib email package.""" + + first_line = True + + def feed(self, data): + """Feed some data, but chomp the first line.""" + if self.first_line: + self.first_line = False + return + parser.FeedParser.feed(self, data) + + +def urlopen(request, data=None): + """Open a given url using curl.""" + if isinstance(request, basestring): + request = Request(request, data) + + curl = pycurl.Curl() + try: + request_headers = [] + if isinstance(request.url, unicode): + request.url = request.url.encode("utf-8") + curl.setopt(pycurl.URL, request.url) + if request.headers: + for key, value in request.headers.items(): + request_headers.append("%s: %s" % (key, value)) + curl.setopt(pycurl.HTTPHEADER, request_headers) + response = Response(request.url) + response_headers_parser = HeaderParser() + curl.setopt(pycurl.WRITEFUNCTION, response.write) + curl.setopt(pycurl.HEADERFUNCTION, response_headers_parser.feed) + curl.setopt(pycurl.FOLLOWLOCATION, 1) + curl.setopt(pycurl.MAXREDIRS, 5) + curl.setopt(pycurl.SSL_VERIFYPEER, 1) + curl.setopt(pycurl.SSL_VERIFYHOST, 2) + if request.data: + curl.setopt(pycurl.POST, 1) + curl.setopt(pycurl.POSTFIELDS, request.data) + curl.perform() + code = curl.getinfo(pycurl.HTTP_CODE) + response_headers = response_headers_parser.close() + response.finish(code, response_headers) + except pycurl.error as e: + raise HTTPError(request.url, e[0], curl.errstr()) + else: + if code in (200, 0): + return response + else: + if code == 401: + errorclass = UnauthorizedError + else: + errorclass = HTTPError + message = httplib.responses.get(code, "Unknown error") + raise errorclass(request.url, code, message, response) + finally: + curl.close() === added file 'ubuntu_sso/utils/tests/test_curllib.py' --- ubuntu_sso/utils/tests/test_curllib.py 1970-01-01 00:00:00 +0000 +++ ubuntu_sso/utils/tests/test_curllib.py 2011-12-15 18:31:17 +0000 @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- + +# Author: Alejandro J. Cura +# +# Copyright 2011 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 3, as published +# by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +"""Tests for the pycurl based web client.""" + +import cgi +import urllib + +from twisted.application import internet, service +from twisted.internet import defer, threads +from twisted.trial.unittest import TestCase +from twisted.web import http, resource, server + +from ubuntu_sso.utils import curllib + + +SAMPLE_URL = "http://protocultura.net/" +SAMPLE_KEY = "result" +SAMPLE_VALUE = "sample result" +SAMPLE_RESOURCE = '{"%s": "%s"}' % (SAMPLE_KEY, SAMPLE_VALUE) +SAMPLE_CREDENTIALS = dict( + consumer_key="consumer key", + consumer_secret="consumer secret", + token="the real token", + token_secret="the token secret", +) +SAMPLE_HEADERS = {SAMPLE_KEY: SAMPLE_VALUE} +SAMPLE_PARAMS = SAMPLE_HEADERS +SAMPLE_RAW_HEADERS = [ + "HTTP/1.1 200 OK\r\n", + "%s: %s\r\n" % (SAMPLE_KEY, SAMPLE_VALUE), + "Multiline: This\r\n", + " is a sample\r\n", + " multiline header.\r\n", + "\r\n", +] + + +# pylint: disable=C0103 +# t.w.resource methods have freeform cased names + +class MockResource(resource.Resource): + """A simple web resource.""" + isLeaf = True + contents = "" + + def getChild(self, name, request): + """Get a given child resource.""" + if name == '': + return self + return resource.Resource.getChild(self, name, request) + + def render_GET(self, request): + """Make a bit of html out of these resource's content.""" + return self.contents + + +class VerifyHeadersResource(resource.Resource): + """A resource that verifies the headers received.""" + + def render_GET(self, request): + """Make a bit of html out of these resource's content.""" + headers = request.requestHeaders.getRawHeaders(SAMPLE_KEY) + if headers != [SAMPLE_VALUE]: + request.setResponseCode(http.BAD_REQUEST) + return "ERROR: Expected header not present." + return SAMPLE_RESOURCE + + +class VerifyPostResource(resource.Resource): + """A resource that verifies the post was .""" + + def render_POST(self, request): + """Make a bit of html out of these resource's content.""" + values = cgi.escape(request.args[SAMPLE_KEY][0]) + if values != SAMPLE_VALUE: + request.setResponseCode(http.BAD_REQUEST) + return "ERROR: Expected value not present." + return SAMPLE_RESOURCE + + +class HeadedResource(resource.Resource): + """A resource that sends some response headers.""" + + def render_GET(self, request): + """Make a bit of html out of these resource's content.""" + request.setHeader(SAMPLE_KEY, SAMPLE_VALUE) + return SAMPLE_RESOURCE + +# pylint: enable=C0103 + + +class MockWebService(object): + """A mock webservice for testing""" + + def __init__(self): + """Start up this instance.""" + root = resource.Resource() + mock_resource = MockResource() + mock_resource.contents = SAMPLE_RESOURCE + root.putChild("mock_resource", mock_resource) + root.putChild("throwerror", resource.NoResource()) + unauthorized = resource.ErrorPage(resource.http.UNAUTHORIZED, + "Unauthorized", "Unauthorized") + root.putChild("unauthorized", unauthorized) + root.putChild("verifyheaders", VerifyHeadersResource()) + root.putChild("verifypost", VerifyPostResource()) + root.putChild("headed_resource", HeadedResource()) + + site = server.Site(root) + application = service.Application('web') + self.service_collection = service.IServiceCollection(application) + #pylint: disable=E1101 + self.tcpserver = internet.TCPServer(0, site) + self.tcpserver.setServiceParent(self.service_collection) + self.service_collection.startService() + + def get_url(self): + """Build the url for this mock server.""" + #pylint: disable=W0212 + port_num = self.tcpserver._port.getHost().port + return "http://localhost:%d/" % port_num + + def stop(self): + """Shut it down.""" + #pylint: disable=E1101 + return self.service_collection.stopService() + + +class HeaderParserCase(TestCase): + """Test the HeaderParser class.""" + + def parse(self, lines): + """Feed a parser with some lines, and return the result.""" + parser = curllib.HeaderParser() + for line in lines: + parser.feed(line) + return parser.close() + + def test_skips_first_element(self): + """It skips the first element (the status code).""" + test_headers = SAMPLE_RAW_HEADERS[0:1] + result = self.parse(test_headers) + self.assertEqual(len(result.keys()), 0) + + def test_skips_last_element(self): + """It skips the last element (an empty line with CR/LF).""" + test_headers = [SAMPLE_RAW_HEADERS[0], SAMPLE_RAW_HEADERS[-1]] + result = self.parse(test_headers) + self.assertEqual(len(result.keys()), 0) + + def test_parses_the_rest(self): + """It parses all the rest.""" + result = self.parse(SAMPLE_RAW_HEADERS) + self.assertEqual(len(result.keys()), 2) + self.assertEqual(result[SAMPLE_KEY], SAMPLE_VALUE) + + +class CurllibTestCase(TestCase): + """Tests for the curllib.""" + + @defer.inlineCallbacks + def setUp(self): + """Initialize this testcase.""" + yield super(CurllibTestCase, self).setUp() + self.ws = MockWebService() + self.url = self.ws.get_url() + self.addCleanup(self.ws.stop) + + def urlopen_in_thread(self, *args, **kwargs): + """Run curllib in a thread, so it doesn't block the mock webserver.""" + return threads.deferToThread(curllib.urlopen, *args, **kwargs) + + @defer.inlineCallbacks + def test_urlopen(self): + """Test a simple urlopen.""" + response = yield self.urlopen_in_thread(self.url + "mock_resource") + self.assertEqual(response.read(), SAMPLE_RESOURCE) + + @defer.inlineCallbacks + def test_urlopen_unicode(self): + """Test an unicode url.""" + url_path = u"mock_resource?test=ñandú" + response = yield self.urlopen_in_thread(self.url + url_path) + self.assertEqual(response.read(), SAMPLE_RESOURCE) + + @defer.inlineCallbacks + def test_urlopen_receiving_headers(self): + """Test urlopen receiving headers.""" + response = yield self.urlopen_in_thread(self.url + "headed_resource") + self.assertEqual(response.headers[SAMPLE_KEY], SAMPLE_VALUE) + + @defer.inlineCallbacks + def test_urlopen_sending_headers(self): + """Test urlopen sending headers.""" + request = curllib.Request(self.url + "verifyheaders", + headers=SAMPLE_HEADERS) + response = yield self.urlopen_in_thread(request) + self.assertEqual(SAMPLE_RESOURCE, response.read()) + + @defer.inlineCallbacks + def test_urlopen_post_parameters(self): + """Test urlopen with POST parameters.""" + data = urllib.urlencode(SAMPLE_PARAMS) + request = curllib.Request(self.url + "verifypost", data=data) + response = yield self.urlopen_in_thread(request) + self.assertEqual(SAMPLE_RESOURCE, response.read()) + + @defer.inlineCallbacks + def test_urlopen_unauthorized(self): + """Test urlopen with unauthorized urls.""" + d = self.urlopen_in_thread(self.url + "unauthorized") + e = yield self.assertFailure(d, curllib.UnauthorizedError) + self.assertEqual(e.code, 401) + + @defer.inlineCallbacks + def test_urlopen_some_other_error(self): + """Test urlopen with some other error.""" + d = self.urlopen_in_thread(self.url + "throwerror") + e = yield self.assertFailure(d, curllib.HTTPError) + self.assertEqual(e.code, 404) + + @defer.inlineCallbacks + def test_connection_failure(self): + """Test a failure to connect.""" + invalid_url = "http://localhost:99999/" # the port is way over 65535! + d = self.urlopen_in_thread(invalid_url) + e = yield self.assertFailure(d, curllib.HTTPError) + self.assertEqual(e.code, curllib.pycurl.E_URL_MALFORMAT) + + +class ResponseTestCase(TestCase): + """Tests for the Response class.""" + + def test_rewinds_on_finish(self): + """The buffer is rewinded when the response is finished.""" + response = curllib.Response(SAMPLE_URL) + response.write(SAMPLE_KEY) + response.write(SAMPLE_VALUE) + response.finish(200, {}) + self.assertEqual(response.read(), SAMPLE_KEY + SAMPLE_VALUE) + + +class RequestTestCase(TestCase): + """Tests for the Request class.""" + + def test_get_full_url(self): + """Test the get_full_url method.""" + request = curllib.Request(SAMPLE_URL) + self.assertEqual(request.get_full_url(), SAMPLE_URL) + + +class FakeCurl(object): + """A fake Curl that records options set.""" + + def __init__(self): + """Initialize this fake.""" + self.options = {} + + def setopt(self, key, value): + """Save a copy of the option.""" + self.options[key] = value + + def getinfo(self, key): + """Fake a finished operation.""" + if key == curllib.pycurl.HTTP_CODE: + return 200 + + def perform(self): + """Do nothing.""" + + def close(self): + """Do nothing, too.""" + + +class SslVerificationTestCase(TestCase): + """Tests the curllib SSL verification.""" + + def test_ssl_is_verified(self): + """The ssl verification flags are set on the curl object.""" + fake_curl = FakeCurl() + self.patch(curllib.pycurl, "Curl", lambda: fake_curl) + curllib.urlopen("http://localhost:1234") + self.assertEqual(fake_curl.options[curllib.pycurl.SSL_VERIFYPEER], 1) + self.assertEqual(fake_curl.options[curllib.pycurl.SSL_VERIFYHOST], 2)