locust: locust timeout does not work as expected

Describe the bug

I am running locust without the ui in a and in a decentralized manner (about 25 EC2s) dockerized environment, my command is: docker run --log-opt max-size=10m --log-opt max-file=2 -p 8089:8089 -p 5557:5557 -v $PWD/:/mnt/ locust_client -f /mnt/performance_test.py -P 8089 --headless --csv=/mnt/currentTestData --test_file_name currentTestData --use_ui false -u 2850 --user_amount 2850 --host_name $HOST --run-time 8h --host $HOST -r 10 -L INFO

instead of running for 8hours it ends after about 22 minutes give or take, constantly. locust version for the dockerfile is: 1.4.4

I would expect the test to run for 8 hours.

  • OS: Linux (amazon)
  • Python version: 2.7.8
  • Locust version: 1.4.4
  • Locust file contents (anonymized if necessary): ################################
from locust import User, task, between, events, constant
import hashlib
import logging
from endpoint_user import *
from test_metrics_analyser import *
from test_report_generator import *


@events.init_command_line_parser.add_listener
def init_parser(parser):
    parser.add_argument(
        '--wavefront_api_key',
        help="wavefront's api key"
    )
    parser.add_argument(
        '--test_file_name',
        help="the test result file name"
    )
    parser.add_argument(
        '--user_amount',
        help="the amount of users for the test"
    )
    parser.add_argument(
        '--host_name',
        help="the vkm's address"
    )
    parser.add_argument(
        '--use_ui',
        help="use user interface or not"
    )


@events.test_stop.add_listener
def analyse_test_results_and_publish_report(environment, **kwargs):
    logging.info("locust test finished, starting to analyse test results")


class LongRunTest(EndpointUser):
    wait_time = constant(0.1)
    secret_name = "secret_for_role_protected_folder"
    folder_name = "dir_in_root" + generate_random_alphanum_string(12)
    secret_item = folder_name + "/" + secret_name

    def __init__(self, *args, **kwargs):
        super(LongRunTest, self).__init__(*args, **kwargs)
        self.create_role_protected_folder()
        self.create_secret_in_role_protected_folder()
        self.create_key_in_role_protected_folder()
        self.create_derived_key()

    def __del__(self):
        #self.cleanup()
        logging.info("Cleanup Ended")

    @task(4)
    def get_secret(self):
        # logging.info("getting secret: " + self.secret_item)
        self.client.get_item(self.secret_item)

    @task(8)
    def twice_derive_key(self):
        self.client.rcc_derive_twice_key(self.key.name, "idps-team", version=self.key.version)
        # logging.info("twice_derive_key ended")

    @task(8)
    def derive_key(self):
        self.client.rcc_derive_key(self.derived_key.name, self.header_data, version=self.derived_key.version)
        # logging.info("RCC derive key ended")

    @task(1)
    def get_item_list_unprotected_folder(self):
        self.client.get_items_list(self.folder_name)
        # logging.info("Get item list for an unprotected folder finished")

    @task(1)
    def get_non_existent_item_from_existing_folder(self):
        non_existent_secret_name = self.folder_name + "/" + "non_existing_secret"
        self.client.get_item_nonexistent(non_existent_secret_name)
        # logging.info("Get non-existing item in an existing role-protected folder finished")


    def cleanup(self):
        logging.info("Cleanup: Test ended, starting cleanup")
        self.client.ep.clear_item(self.secret_item)
        logging.info("Cleanup: secret cleared: " + self.secret_item)
        self.client.ep.clear_item(self.key.name)
        logging.info("Cleanup: key cleared " + self.key.name)
        # self.client.ep.clear_item(self.derived_key.name)
        # logging.info("Cleanup: key cleared "+self.derived_key.name)
        self.client.ep.clear_item(self.folder_name)
        logging.info("Cleanup: folder cleared " + self.folder_name)

About this issue

  • Original URL
  • State: closed
  • Created 3 years ago
  • Comments: 15 (5 by maintainers)

Most upvoted comments

well I tried cutting it down, and all of a sudden everything started to work, not sure what was the problem. thank you for you replies, if I will be able to replicate the problem I will come back here with the replicated problem.