mrq/tests/test_memoryleaks.py at 80eb6f20dad091f6a8439e7bbab86e172ed5d956 · pricingassistant/mrq · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import time


def get_diff_after_jobs(worker, n_tasks, leak, sleep=0):

    time.sleep(3)

    mem_start = worker.get_report()["process"]["mem"]["rss"]

    worker.send_tasks(
        "tests.tasks.general.Leak",
        [{"size": leak, "sleep": sleep} for _ in range(n_tasks)],
        queue="default",
        block=True
    )

    time.sleep(3)

    mem_stop = worker.get_report()["process"]["mem"]["rss"]

    diff = mem_stop - mem_start

    print "Memory diff for %s tasks was %s" % (n_tasks, diff)

    return diff


def test_memoryleaks_noleak(worker):

    worker.start(
        flags="--trace_memory --gevent 1 --mongodb_logs 0 --report_interval 10000")

    # Send it once to add to imports
    get_diff_after_jobs(worker, 10, 0)

    diff100 = get_diff_after_jobs(worker, 100, 0)

    assert worker.mongodb_jobs.mrq_jobs.count() == 100 + 10

    diff200 = get_diff_after_jobs(worker, 200, 0)

    assert worker.mongodb_jobs.mrq_jobs.count() == 200 + 100 + 10

    # Most of the tasks should have mem_diff == 0
    assert worker.mongodb_jobs.mrq_jobs.find(
        {"memory_diff": 0}).count() > 310 * 0.95

    assert abs(diff100 - diff200) < 20000

    assert diff200 < 150000


def test_memoryleaks_1mleak(worker):

    worker.start(
        flags="--trace_memory --gevent 1 --mongodb_logs 0 --report_interval 10000")

    # Send it once to add to imports
    get_diff_after_jobs(worker, 10, 0)
    assert worker.mongodb_jobs.mrq_jobs.find({"memory_diff": 0}).count() > 5

    worker.mongodb_jobs.mrq_jobs.remove()

    # 1M leak!
    # sleep is needed so that psutil measurements are accurate :-/
    diff1m = get_diff_after_jobs(worker, 10, 100000, sleep=0)

    assert diff1m > 900000

    assert worker.mongodb_jobs.mrq_jobs.find(
        {"memory_diff": {"$gte": 80000}}).count() == 10