From c31ac7fe6194b6de4ba666e281df5acc343e798e Mon Sep 17 00:00:00 2001 From: Adrien Chauve Date: Fri, 23 Jan 2015 15:42:13 +0100 Subject: [PATCH 1/4] Make worker maximum sleep time configurable --- mrq/config.py | 8 ++++++++ mrq/worker.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mrq/config.py b/mrq/config.py index fd693b03..41abb18c 100644 --- a/mrq/config.py +++ b/mrq/config.py @@ -296,6 +296,14 @@ def add_parser_args(parser, config_type): type=str, help='Overwrite the local IP, to be displayed in the dashboard.') + parser.add_argument( + '--max_sleep', + default=1., + type=float, + action='store', + help='Max seconds while worker may sleep waiting for a new job. ' + 'Can be < 1.') + def get_config( sources=( diff --git a/mrq/worker.py b/mrq/worker.py index 18ce451e..96a160f3 100644 --- a/mrq/worker.py +++ b/mrq/worker.py @@ -408,7 +408,7 @@ def work_loop(self): if len(jobs) < free_pool_slots: self.status = "wait" wait_count += 1 - gevent.sleep(min(1, 0.001 * wait_count)) + gevent.sleep(min(self.config["max_sleep"], 0.001 * wait_count)) except StopRequested: pass From 826ccbd04c167d7cef9ec32d5dca72747d803900 Mon Sep 17 00:00:00 2001 From: Adrien Chauve Date: Fri, 23 Jan 2015 15:42:32 +0100 Subject: [PATCH 2/4] Ignore .idea/ files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f768ad1d..03e1bc05 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,8 @@ nosetests.xml .project .pydevproject +.idea/ + venv pypy .DS_Store From c8f3ce623e5fb8824e8c4df70bd69c84293985b0 Mon Sep 17 00:00:00 2001 From: Adrien Chauve Date: Fri, 23 Jan 2015 15:54:41 +0100 Subject: [PATCH 3/4] Remove log as it is very verbose; find better solution? --- mrq/worker.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mrq/worker.py b/mrq/worker.py index 96a160f3..a79ca16e 100644 --- a/mrq/worker.py +++ b/mrq/worker.py @@ -375,11 +375,6 @@ def work_loop(self): self.status = "full" gevent.sleep(0.01) - quiet = not (wait_count % 20 == 0 or wait_count > 1000) - - if not quiet: - self.log.info('Fetching %s jobs from %s' % (free_pool_slots, self.queues)) - jobs = [] for queue_name in self.queues: From 86a0a05ee57d6ff1eb320492a4f5f2797db735ae Mon Sep 17 00:00:00 2001 From: Adrien Chauve Date: Tue, 3 Feb 2015 16:13:02 +0100 Subject: [PATCH 4/4] Rename max_sleep option to max_latency --- mrq/config.py | 2 +- mrq/worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mrq/config.py b/mrq/config.py index 41abb18c..aed2a767 100644 --- a/mrq/config.py +++ b/mrq/config.py @@ -297,7 +297,7 @@ def add_parser_args(parser, config_type): help='Overwrite the local IP, to be displayed in the dashboard.') parser.add_argument( - '--max_sleep', + '--max_latency', default=1., type=float, action='store', diff --git a/mrq/worker.py b/mrq/worker.py index a79ca16e..732ed975 100644 --- a/mrq/worker.py +++ b/mrq/worker.py @@ -403,7 +403,7 @@ def work_loop(self): if len(jobs) < free_pool_slots: self.status = "wait" wait_count += 1 - gevent.sleep(min(self.config["max_sleep"], 0.001 * wait_count)) + gevent.sleep(min(self.config["max_latency"], 0.001 * wait_count)) except StopRequested: pass