ClusterOperationsLoadTest.test_put_deploy_selected_nodes is running longer then expected
https://ci.fuel-infra.org/job/nailgun_performance_tests/308/console
=================================== FAILURES ===================================
___________ ClusterOperationsLoadTest.test_put_deploy_selected_nodes ___________
args = (<nailgun.test.performance.unit.test_cluster_operations.ClusterOperationsLoadTest testMethod=test_put_deploy_selected_nodes>,)
kwargs = {}
test = <nailgun.test.performance.unit.test_cluster_operations.ClusterOperationsLoadTest testMethod=test_put_deploy_selected_nodes>
number_of_runs = 10, run = 9, compare_with_previous = False
class_name = 'ClusterOperationsLoadTest', previous_test_results = {}
current_rest_results = defaultdict(<function <lambda> at 0x5e93b90>, {'1': defaultdict(<type 'list'>,...272289991378784, 11.296671152114868], 'request_name': 'DeploySelectedNodes'})})
call_number = '1'
results = defaultdict(<type 'list'>, {'expect_time': 10, 'results': [11.261782884597778,...1.272289991378784, 11.296671152114868], 'request_name': 'DeploySelectedNodes'})
request_name = 'DeploySelectedNodes'
@functools.wraps(f)
def wrapper(*args, **kwargs):
# read results of previous correct run
test = args[0]
number_of_runs = settings.PERFORMANCE_TESTS_RUN_NUMBER
# run tests multiple time to get more exact results
for run in six.moves.range(number_of_runs):
f(*args, **kwargs)
copy_test_results(run)
# reset call number for unittests
test.call_number = 1
compare_with_previous = False
class_name = test.__class__.__name__
previous_test_results = read_previous_results().get(class_name, {}).\
get(str(test), {})
current_rest_results = test.tests_results[str(test)]
if len(previous_test_results) == len(current_rest_results):
compare_with_previous = True
for call_number, results in six.iteritems(
test.tests_results[str(test)]):
request_name = results['request_name']
# normalize results and compute avg
normalized = normalize(results['results'], 0.025)
avg_time = sum(normalized) / len(normalized)
# check if previous results exists
prev_time = None
if compare_with_previous:
if request_name in \
previous_test_results[call_number]['request_name']:
# we give some % (default 10%) of tolerance for previous
# expected time
prev_time = (
previous_test_results[call_number]['expect_time'] *
(1.0 + settings.PERFORMANCE_TESTS_TOLERANCE))
expect_time = prev_time or results['expect_time']
test.tests_results[str(test)]
test.assertTrue(
avg_time <= expect_time,
"Average execution time: {exec_time} is greater, "
"than expected: {max_exec_time}".format(
exec_time=avg_time,
> max_exec_time=expect_time))
E AssertionError: False is not True : Average execution time: 11.269977755 is greater, than expected: 10
nailgun/test/performance/base.py:360: AssertionError
generated xml file: /home/jenkins/workspace/nailgun_performance_tests/nailgun.xml
Fix proposed to branch: master
Review: https:/