=== modified file 'bin/ubuntuone-preferences' --- bin/ubuntuone-preferences 2010-06-07 17:28:04 +0000 +++ bin/ubuntuone-preferences 2012-05-23 00:26:57 +0000 @@ -38,9 +38,10 @@ from ubuntuone.syncdaemon.tools import SyncDaemonTool from ubuntuone.logger import (basic_formatter, LOGFOLDER, CustomRotatingFileHandler) +from ubuntuone.utils import curllib import logging -import httplib, urlparse, socket +import urlparse, socket import dbus.service from dbus.exceptions import DBusException @@ -143,20 +144,15 @@ oauth_request.sign_request(oauth.OAuthSignatureMethod_HMAC_SHA1(), consumer, token) - scheme, netloc, path, query, fragment = urlparse.urlsplit(url) + request = curllib.Request(url, headers=oauth_request.to_header()) - conn = httplib.HTTPSConnection(netloc) try: - conn.request(method, path, headers=oauth_request.to_header()) - response = conn.getresponse() # shouldn't block - if response.status == 200: - data = response.read() # neither should this - result = simplejson.loads(data) - else: - result = {'status' : response.status, - 'reason' : response.reason} - except socket.error, e: - result = {'error' : e} + response = curllib.urlopen(request) # shouldn't block + data = response.read() # neither should this + result = simplejson.loads(data) + except curllib.HTTPError, e: + result = {'status' : e.message, + 'reason' : e.code} gtk.gdk.threads_enter() callback(result) === modified file 'data/syncdaemon.conf' --- data/syncdaemon.conf 2010-04-23 15:23:09 +0000 +++ data/syncdaemon.conf 2012-05-22 02:46:16 +0000 @@ -2,7 +2,7 @@ host.default = fs-1.one.ubuntu.com host.help = The server address -dns_srv.default = _https._tcp.fs.ubuntuone.com +dns_srv.default = _https._tcp.fs.one.ubuntu.com dns_srv.help = The DNS SRV record disable_ssl_verify.default = False === modified file 'tests/syncdaemon/test_action_queue.py' --- tests/syncdaemon/test_action_queue.py 2010-05-17 22:23:56 +0000 +++ tests/syncdaemon/test_action_queue.py 2011-12-15 20:05:29 +0000 @@ -25,7 +25,6 @@ import os import shutil import unittest -import urllib2 import uuid import dbus @@ -52,6 +51,7 @@ ) from ubuntuone.syncdaemon.event_queue import EventQueue, EVENTS from ubuntuone.syncdaemon.volume_manager import UDF +from ubuntuone.utils import curllib DBusInterface.test = True @@ -1136,7 +1136,7 @@ try: res = self.command._change_public_access_http() finally: - action_queue.urlopen = urllib2.urlopen + action_queue.urlopen = curllib.urlopen self.assertEqual( {'is_public': True, 'public_url': 'http://example.com'}, res) @@ -1154,7 +1154,7 @@ def test_handle_failure_push_event(self): """Test AQ_CHANGE_PUBLIC_ACCESS_ERROR is pushed on failure.""" msg = 'Something went wrong' - failure = Failure(urllib2.HTTPError( + failure = Failure(curllib.HTTPError( "http://example.com", 500, "Error", [], StringIO(msg))) res = self.command.handle_failure(failure=failure) events = [('AQ_CHANGE_PUBLIC_ACCESS_ERROR', (), @@ -1233,7 +1233,7 @@ try: res = self.command._get_public_files_http() finally: - action_queue.urlopen = urllib2.urlopen + action_queue.urlopen = curllib.urlopen self.assertEqual([{'node_id': str(node_id), 'volume_id': '', 'public_url': 'http://example.com'}, {'node_id': str(node_id_2), @@ -1252,7 +1252,7 @@ def test_handle_failure_push_event(self): """Test AQ_PUBLIC_FILES_LIST_ERROR is pushed on failure.""" msg = 'Something went wrong' - failure = Failure(urllib2.HTTPError( + failure = Failure(curllib.HTTPError( "http://example.com", 500, "Error", [], StringIO(msg))) res = self.command.handle_failure(failure=failure) events = [('AQ_PUBLIC_FILES_LIST_ERROR', (), {'error': msg})] === modified file 'ubuntuone/oauthdesktop/auth.py' --- ubuntuone/oauthdesktop/auth.py 2010-02-10 17:35:26 +0000 +++ ubuntuone/oauthdesktop/auth.py 2012-05-23 03:29:00 +0000 @@ -63,6 +63,16 @@ except ImportError: pass else: + def _verify_hostname(hostname, cert): + """Verify the server hostname.""" + # For Ubuntu One certificates only the commonName is needed + for subject in cert.get("subject", []): + for key, value in subject: + if key == "commonName": + if value == hostname: + return + raise socket.error("SSL hostname validation failed.") + def _connect_wrapper(self): """Override HTTPSConnection.connect to require certificate checks""" sock = socket.create_connection((self.host, self.port), self.timeout) @@ -75,6 +85,7 @@ self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs="/etc/ssl/certs/ca-certificates.crt") + _verify_hostname(self.host, self.sock.getpeercert()) httplib.HTTPSConnection.connect = _connect_wrapper === modified file 'ubuntuone/syncdaemon/action_queue.py' --- ubuntuone/syncdaemon/action_queue.py 2010-04-12 18:47:01 +0000 +++ ubuntuone/syncdaemon/action_queue.py 2012-05-22 02:18:06 +0000 @@ -39,7 +39,6 @@ from collections import deque, defaultdict from functools import wraps, partial from urllib import urlencode -from urllib2 import urlopen, Request, HTTPError from urlparse import urljoin from zope.interface import implements @@ -57,6 +56,7 @@ from ubuntuone.storageprotocol.context import get_ssl_context from ubuntuone.syncdaemon.interfaces import IActionQueue, IMarker from ubuntuone.syncdaemon.logger import mklog, TRACE +from ubuntuone.utils.curllib import urlopen, Request, HTTPError logger = logging.getLogger("ubuntuone.SyncDaemon.ActionQueue") @@ -802,7 +802,7 @@ def _make_connection(self, result): """Do the real connect call.""" host, port = result - ssl_context = get_ssl_context(self.disable_ssl_verify) + ssl_context = get_ssl_context(self.disable_ssl_verify, host) if self.use_ssl: self.connector = reactor.connectSSL(host, port, factory=self, contextFactory=ssl_context, === modified file 'ubuntuone/u1sync/client.py' --- ubuntuone/u1sync/client.py 2010-02-10 17:35:26 +0000 +++ ubuntuone/u1sync/client.py 2012-05-22 00:03:29 +0000 @@ -393,7 +393,7 @@ """Connect to host/port using ssl.""" def _connect(): """deferred part.""" - ctx = get_ssl_context(no_verify) + ctx = get_ssl_context(no_verify, host) self.reactor.connectSSL(host, port, self.factory, ctx) self._connect_inner(_connect) === added directory 'ubuntuone/utils' === added file 'ubuntuone/utils/__init__.py' --- ubuntuone/utils/__init__.py 1970-01-01 00:00:00 +0000 +++ ubuntuone/utils/__init__.py 2011-12-15 19:57:10 +0000 @@ -0,0 +1,14 @@ +# Copyright 2011 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 3, as published +# by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +"""The ubuntuone.utils package""" === added file 'ubuntuone/utils/curllib.py' --- ubuntuone/utils/curllib.py 1970-01-01 00:00:00 +0000 +++ ubuntuone/utils/curllib.py 2011-12-15 19:57:47 +0000 @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- + +# Author: Alejandro J. Cura +# +# Copyright 2011 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 3, as published +# by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +"""Web client based on pycurl, with an API similar to urllib2.""" + +import httplib + +from email import parser +from StringIO import StringIO + +import pycurl + + +class HTTPError(Exception): + """Error that happens while doing a web request.""" + + def __init__(self, url, code, message, headers=None, fp=None): + """Initialize this instance.""" + super(HTTPError, self).__init__() + self.code = code + self.message = message + if fp: + self.fp = fp + else: + self.fp = StringIO(message) + + def read(self): + """Return the error message.""" + return self.fp.read() + + def __str__(self): + return "<%s: %s>" % (self.code, self.message) + + +class UnauthorizedError(HTTPError): + """An HTTP error when there's an auth problem.""" + + +class Request(object): + """An HTTP request object.""" + + def __init__(self, url, data=None, headers=None): + """Initialize this instance.""" + super(Request, self).__init__() + self.url = url + self.data = data + self.headers = headers + + def get_full_url(self): + """Return the url.""" + return self.url + + def get_data(self): + """Return the data.""" + return self.data + + +class Response(StringIO): + """An HTTP response object.""" + + code = -1 + headers = None + + def __init__(self, url): + """Initialize this instance.""" + StringIO.__init__(self) + self.url = url + + def finish(self, code, headers): + """Finish a response and rewind it.""" + self.code = code + self.headers = headers + self.seek(0) + + +class HeaderParser(parser.FeedParser): + """A parser for the HTTP headers, based on the stdlib email package.""" + + first_line = True + + def feed(self, data): + """Feed some data, but chomp the first line.""" + if self.first_line: + self.first_line = False + return + parser.FeedParser.feed(self, data) + + +def urlopen(request, data=None): + """Open a given url using curl.""" + if isinstance(request, basestring): + request = Request(request, data) + + curl = pycurl.Curl() + try: + request_headers = [] + if isinstance(request.url, unicode): + request.url = request.url.encode("utf-8") + curl.setopt(pycurl.URL, request.url) + if request.headers: + for key, value in request.headers.items(): + request_headers.append("%s: %s" % (key, value)) + curl.setopt(pycurl.HTTPHEADER, request_headers) + response = Response(request.url) + response_headers_parser = HeaderParser() + curl.setopt(pycurl.WRITEFUNCTION, response.write) + curl.setopt(pycurl.HEADERFUNCTION, response_headers_parser.feed) + curl.setopt(pycurl.FOLLOWLOCATION, 1) + curl.setopt(pycurl.MAXREDIRS, 5) + curl.setopt(pycurl.SSL_VERIFYPEER, 1) + curl.setopt(pycurl.SSL_VERIFYHOST, 2) + if request.data: + curl.setopt(pycurl.POST, 1) + curl.setopt(pycurl.POSTFIELDS, request.data) + curl.perform() + code = curl.getinfo(pycurl.HTTP_CODE) + response_headers = response_headers_parser.close() + response.finish(code, response_headers) + except pycurl.error as e: + raise HTTPError(request.url, e[0], curl.errstr()) + else: + if code in (200, 0): + return response + else: + if code == 401: + errorclass = UnauthorizedError + else: + errorclass = HTTPError + message = httplib.responses.get(code, "Unknown error") + raise errorclass(request.url, code, message, response) + finally: + curl.close() === added directory 'ubuntuone/utils/tests' === added file 'ubuntuone/utils/tests/__init__.py' --- ubuntuone/utils/tests/__init__.py 1970-01-01 00:00:00 +0000 +++ ubuntuone/utils/tests/__init__.py 2011-12-15 19:57:03 +0000 @@ -0,0 +1,14 @@ +# Copyright 2011 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 3, as published +# by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +"""Tests for the ubuntuone.utils package""" === added file 'ubuntuone/utils/tests/test_curllib.py' --- ubuntuone/utils/tests/test_curllib.py 1970-01-01 00:00:00 +0000 +++ ubuntuone/utils/tests/test_curllib.py 2011-12-15 19:58:29 +0000 @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- + +# Author: Alejandro J. Cura +# +# Copyright 2011 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 3, as published +# by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +"""Tests for the pycurl based web client.""" + +import cgi +import urllib + +from twisted.application import internet, service +from twisted.internet import defer, threads +from twisted.trial.unittest import TestCase +from twisted.web import http, resource, server + +from ubuntuone.utils import curllib + + +SAMPLE_URL = "http://protocultura.net/" +SAMPLE_KEY = "result" +SAMPLE_VALUE = "sample result" +SAMPLE_RESOURCE = '{"%s": "%s"}' % (SAMPLE_KEY, SAMPLE_VALUE) +SAMPLE_CREDENTIALS = dict( + consumer_key="consumer key", + consumer_secret="consumer secret", + token="the real token", + token_secret="the token secret", +) +SAMPLE_HEADERS = {SAMPLE_KEY: SAMPLE_VALUE} +SAMPLE_PARAMS = SAMPLE_HEADERS +SAMPLE_RAW_HEADERS = [ + "HTTP/1.1 200 OK\r\n", + "%s: %s\r\n" % (SAMPLE_KEY, SAMPLE_VALUE), + "Multiline: This\r\n", + " is a sample\r\n", + " multiline header.\r\n", + "\r\n", +] + + +# pylint: disable=C0103 +# t.w.resource methods have freeform cased names + +class MockResource(resource.Resource): + """A simple web resource.""" + isLeaf = True + contents = "" + + def getChild(self, name, request): + """Get a given child resource.""" + if name == '': + return self + return resource.Resource.getChild(self, name, request) + + def render_GET(self, request): + """Make a bit of html out of these resource's content.""" + return self.contents + + +class VerifyHeadersResource(resource.Resource): + """A resource that verifies the headers received.""" + + def render_GET(self, request): + """Make a bit of html out of these resource's content.""" + headers = request.requestHeaders.getRawHeaders(SAMPLE_KEY) + if headers != [SAMPLE_VALUE]: + request.setResponseCode(http.BAD_REQUEST) + return "ERROR: Expected header not present." + return SAMPLE_RESOURCE + + +class VerifyPostResource(resource.Resource): + """A resource that verifies the post was .""" + + def render_POST(self, request): + """Make a bit of html out of these resource's content.""" + values = cgi.escape(request.args[SAMPLE_KEY][0]) + if values != SAMPLE_VALUE: + request.setResponseCode(http.BAD_REQUEST) + return "ERROR: Expected value not present." + return SAMPLE_RESOURCE + + +class HeadedResource(resource.Resource): + """A resource that sends some response headers.""" + + def render_GET(self, request): + """Make a bit of html out of these resource's content.""" + request.setHeader(SAMPLE_KEY, SAMPLE_VALUE) + return SAMPLE_RESOURCE + +# pylint: enable=C0103 + + +class MockWebService(object): + """A mock webservice for testing""" + + def __init__(self): + """Start up this instance.""" + root = resource.Resource() + mock_resource = MockResource() + mock_resource.contents = SAMPLE_RESOURCE + root.putChild("mock_resource", mock_resource) + root.putChild("throwerror", resource.NoResource()) + unauthorized = resource.ErrorPage(resource.http.UNAUTHORIZED, + "Unauthorized", "Unauthorized") + root.putChild("unauthorized", unauthorized) + root.putChild("verifyheaders", VerifyHeadersResource()) + root.putChild("verifypost", VerifyPostResource()) + root.putChild("headed_resource", HeadedResource()) + + site = server.Site(root) + application = service.Application('web') + self.service_collection = service.IServiceCollection(application) + #pylint: disable=E1101 + self.tcpserver = internet.TCPServer(0, site) + self.tcpserver.setServiceParent(self.service_collection) + self.service_collection.startService() + + def get_url(self): + """Build the url for this mock server.""" + #pylint: disable=W0212 + port_num = self.tcpserver._port.getHost().port + return "http://localhost:%d/" % port_num + + def stop(self): + """Shut it down.""" + #pylint: disable=E1101 + return self.service_collection.stopService() + + +class HeaderParserCase(TestCase): + """Test the HeaderParser class.""" + + def parse(self, lines): + """Feed a parser with some lines, and return the result.""" + parser = curllib.HeaderParser() + for line in lines: + parser.feed(line) + return parser.close() + + def test_skips_first_element(self): + """It skips the first element (the status code).""" + test_headers = SAMPLE_RAW_HEADERS[0:1] + result = self.parse(test_headers) + self.assertEqual(len(result.keys()), 0) + + def test_skips_last_element(self): + """It skips the last element (an empty line with CR/LF).""" + test_headers = [SAMPLE_RAW_HEADERS[0], SAMPLE_RAW_HEADERS[-1]] + result = self.parse(test_headers) + self.assertEqual(len(result.keys()), 0) + + def test_parses_the_rest(self): + """It parses all the rest.""" + result = self.parse(SAMPLE_RAW_HEADERS) + self.assertEqual(len(result.keys()), 2) + self.assertEqual(result[SAMPLE_KEY], SAMPLE_VALUE) + + +class CurllibTestCase(TestCase): + """Tests for the curllib.""" + + @defer.inlineCallbacks + def setUp(self): + """Initialize this testcase.""" + yield super(CurllibTestCase, self).setUp() + self.ws = MockWebService() + self.url = self.ws.get_url() + self.addCleanup(self.ws.stop) + + def urlopen_in_thread(self, *args, **kwargs): + """Run curllib in a thread, so it doesn't block the mock webserver.""" + return threads.deferToThread(curllib.urlopen, *args, **kwargs) + + @defer.inlineCallbacks + def test_urlopen(self): + """Test a simple urlopen.""" + response = yield self.urlopen_in_thread(self.url + "mock_resource") + self.assertEqual(response.read(), SAMPLE_RESOURCE) + + @defer.inlineCallbacks + def test_urlopen_unicode(self): + """Test an unicode url.""" + url_path = u"mock_resource?test=ñandú" + response = yield self.urlopen_in_thread(self.url + url_path) + self.assertEqual(response.read(), SAMPLE_RESOURCE) + + @defer.inlineCallbacks + def test_urlopen_receiving_headers(self): + """Test urlopen receiving headers.""" + response = yield self.urlopen_in_thread(self.url + "headed_resource") + self.assertEqual(response.headers[SAMPLE_KEY], SAMPLE_VALUE) + + @defer.inlineCallbacks + def test_urlopen_sending_headers(self): + """Test urlopen sending headers.""" + request = curllib.Request(self.url + "verifyheaders", + headers=SAMPLE_HEADERS) + response = yield self.urlopen_in_thread(request) + self.assertEqual(SAMPLE_RESOURCE, response.read()) + + @defer.inlineCallbacks + def test_urlopen_post_parameters(self): + """Test urlopen with POST parameters.""" + data = urllib.urlencode(SAMPLE_PARAMS) + request = curllib.Request(self.url + "verifypost", data=data) + response = yield self.urlopen_in_thread(request) + self.assertEqual(SAMPLE_RESOURCE, response.read()) + + @defer.inlineCallbacks + def test_urlopen_unauthorized(self): + """Test urlopen with unauthorized urls.""" + d = self.urlopen_in_thread(self.url + "unauthorized") + e = yield self.assertFailure(d, curllib.UnauthorizedError) + self.assertEqual(e.code, 401) + + @defer.inlineCallbacks + def test_urlopen_some_other_error(self): + """Test urlopen with some other error.""" + d = self.urlopen_in_thread(self.url + "throwerror") + e = yield self.assertFailure(d, curllib.HTTPError) + self.assertEqual(e.code, 404) + + @defer.inlineCallbacks + def test_connection_failure(self): + """Test a failure to connect.""" + invalid_url = "http://localhost:99999/" # the port is way over 65535! + d = self.urlopen_in_thread(invalid_url) + e = yield self.assertFailure(d, curllib.HTTPError) + self.assertEqual(e.code, curllib.pycurl.E_URL_MALFORMAT) + + +class ResponseTestCase(TestCase): + """Tests for the Response class.""" + + def test_rewinds_on_finish(self): + """The buffer is rewinded when the response is finished.""" + response = curllib.Response(SAMPLE_URL) + response.write(SAMPLE_KEY) + response.write(SAMPLE_VALUE) + response.finish(200, {}) + self.assertEqual(response.read(), SAMPLE_KEY + SAMPLE_VALUE) + + +class RequestTestCase(TestCase): + """Tests for the Request class.""" + + def test_get_full_url(self): + """Test the get_full_url method.""" + request = curllib.Request(SAMPLE_URL) + self.assertEqual(request.get_full_url(), SAMPLE_URL) + + +class FakeCurl(object): + """A fake Curl that records options set.""" + + def __init__(self): + """Initialize this fake.""" + self.options = {} + + def setopt(self, key, value): + """Save a copy of the option.""" + self.options[key] = value + + def getinfo(self, key): + """Fake a finished operation.""" + if key == curllib.pycurl.HTTP_CODE: + return 200 + + def perform(self): + """Do nothing.""" + + def close(self): + """Do nothing, too.""" + + +class SslVerificationTestCase(TestCase): + """Tests the curllib SSL verification.""" + + def test_ssl_is_verified(self): + """The ssl verification flags are set on the curl object.""" + fake_curl = FakeCurl() + self.patch(curllib.pycurl, "Curl", lambda: fake_curl) + curllib.urlopen("http://localhost:1234") + self.assertEqual(fake_curl.options[curllib.pycurl.SSL_VERIFYPEER], 1) + self.assertEqual(fake_curl.options[curllib.pycurl.SSL_VERIFYHOST], 2)