diff --git a/test/probe/common.py b/test/probe/common.py index 467598c..8398463 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -20,6 +20,8 @@ import sys from time import sleep, time from collections import defaultdict import unittest +from hashlib import md5 +from uuid import uuid4 from nose import SkipTest from swiftclient import get_auth, head_account @@ -254,6 +256,49 @@ def get_policy(**kwargs): raise SkipTest('No policy matching %s' % kwargs) +class Body(object): + + def __init__(self, total=3.5 * 2 ** 20): + self.length = total + self.hasher = md5() + self.read_amount = 0 + self.chunk = uuid4().hex * 2 ** 10 + self.buff = '' + + @property + def etag(self): + return self.hasher.hexdigest() + + def __len__(self): + return self.length + + def read(self, amount): + if len(self.buff) < amount: + try: + self.buff += next(self) + except StopIteration: + pass + rv, self.buff = self.buff[:amount], self.buff[amount:] + return rv + + def __iter__(self): + return self + + def next(self): + if self.buff: + rv, self.buff = self.buff, '' + return rv + if self.read_amount >= self.length: + raise StopIteration() + rv = self.chunk[:int(self.length - self.read_amount)] + self.read_amount += len(rv) + self.hasher.update(rv) + return rv + + def __next__(self): + return next(self) + + class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 37fb762..a57ee87 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -16,13 +16,16 @@ from unittest import main from uuid import uuid4 +import random +from hashlib import md5 from swiftclient import client from swift.common import direct_client from swift.common.exceptions import ClientException from swift.common.manager import Manager -from test.probe.common import kill_server, ReplProbeTest, start_server +from test.probe.common import (kill_server, start_server, ReplProbeTest, + ECProbeTest, Body) class TestObjectHandoff(ReplProbeTest): @@ -211,5 +214,67 @@ class TestObjectHandoff(ReplProbeTest): self.fail("Expected ClientException but didn't get it") +class TestECObjectHandoffOverwrite(ECProbeTest): + + def get_object(self, container_name, object_name): + headers, body = client.get_object(self.url, self.token, + container_name, + object_name, + resp_chunk_size=64 * 2 ** 10) + resp_checksum = md5() + for chunk in body: + resp_checksum.update(chunk) + return resp_checksum.hexdigest() + + def test_ec_handoff_overwrite(self): + container_name = 'container-%s' % uuid4() + object_name = 'object-%s' % uuid4() + + # create EC container + headers = {'X-Storage-Policy': self.policy.name} + client.put_container(self.url, self.token, container_name, + headers=headers) + + # PUT object + old_contents = Body() + client.put_object(self.url, self.token, container_name, + object_name, contents=old_contents) + + # get our node lists + opart, onodes = self.object_ring.get_nodes( + self.account, container_name, object_name) + + # shutdown one of the primary data nodes + failed_primary = random.choice(onodes[:self.policy.ec_ndata]) + failed_primary_device_path = self.device_dir('object', failed_primary) + self.kill_drive(failed_primary_device_path) + + # overwrite our object with some new data + new_contents = Body() + client.put_object(self.url, self.token, container_name, + object_name, contents=new_contents) + + # restore failed primary device + self.revive_drive(failed_primary_device_path) + + # sanity - failed node has old contents + req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)} + headers = direct_client.direct_head_object( + failed_primary, opart, self.account, container_name, + object_name, headers=req_headers) + self.assertEqual(headers['X-Object-Sysmeta-EC-Etag'], + old_contents.etag) + + # clear node error limiting + Manager(['proxy']).restart() + + # read object + for i in range(10): + # do it a few times to make sure node shuffle doesn't + # accidently work around the issue for us + resp_etag = self.get_object(container_name, object_name) + self.assertEqual(resp_etag, new_contents.etag) + + if __name__ == '__main__': main() diff --git a/test/probe/test_reconstructor_revert.py b/test/probe/test_reconstructor_revert.py index 1daf7a3..e04894b 100755 --- a/test/probe/test_reconstructor_revert.py +++ b/test/probe/test_reconstructor_revert.py @@ -21,7 +21,7 @@ import random import shutil from collections import defaultdict -from test.probe.common import ECProbeTest +from test.probe.common import ECProbeTest, Body from swift.common import direct_client from swift.common.storage_policy import EC_POLICY @@ -31,32 +31,6 @@ from swift.obj import reconstructor from swiftclient import client -class Body(object): - - def __init__(self, total=3.5 * 2 ** 20): - self.total = total - self.hasher = md5() - self.size = 0 - self.chunk = 'test' * 16 * 2 ** 10 - - @property - def etag(self): - return self.hasher.hexdigest() - - def __iter__(self): - return self - - def next(self): - if self.size > self.total: - raise StopIteration() - self.size += len(self.chunk) - self.hasher.update(self.chunk) - return self.chunk - - def __next__(self): - return next(self) - - class TestReconstructorRevert(ECProbeTest): def setUp(self):