Update dependencies & Docker environment. Fixes #225

Author: Sylvain Zimmer

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:jessie
+FROM debian:buster-slim
 
 #
 # httpredir.debian.org is often unreliable
@@ -11,32 +11,35 @@ FROM debian:jessie
 #     deb http://security.debian.org jessie/updates main\n' \
 #     > /etc/apt/sources.list
 
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
-RUN echo "deb http://repo.mongodb.org/apt/debian jessie/mongodb-org/3.4 main" > /etc/apt/sources.list.d/mongodb-org-3.4.list
 RUN apt-get update && \
 	apt-get install -y --no-install-recommends \
 				curl \
 				gcc \
 				python-dev \
 				python-pip \
+				python-setuptools \
 				python3-pip \
     			python3-dev \
+    			python3-setuptools \
     			make \
     			git \
     			vim \
     			bzip2 \
-				mongodb-org \
 				nginx redis-server \
 				g++ \
 	&& \
 	apt-get clean -y && \
 	rm -rf /var/lib/apt/lists/*
 
-RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
 RUN apt-get install -y --no-install-recommends nodejs
 
+RUN curl -sL https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add -
+RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.2 main" > /etc/apt/sources.list.d/mongodb-org-4.2.list
+RUN apt-get update && apt-get install -y --no-install-recommends mongodb-org
+
 # Download pypy
-RUN curl -sL 'https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-5.8-1-linux_x86_64-portable.tar.bz2' > /pypy.tar.bz2 && tar jxvf /pypy.tar.bz2 && rm -rf /pypy.tar.bz2 && mv /pypy-* /pypy
+RUN curl -sL 'https://github.com/squeaky-pl/portable-pypy/releases/download/pypy-7.2.0/pypy-7.2.0-linux_x86_64-portable.tar.bz2' > /pypy.tar.bz2 && tar jxvf /pypy.tar.bz2 && rm -rf /pypy.tar.bz2 && mv /pypy* /pypy
 
 # Upgrade pip
 RUN pip install --upgrade --ignore-installed pip
@@ -48,16 +51,16 @@ ADD requirements-base.txt /app/requirements-base.txt
 ADD requirements-dev.txt /app/requirements-dev.txt
 ADD requirements-dashboard.txt /app/requirements-dashboard.txt
 
-RUN pip3 install -r /app/requirements-heroku.txt && \
-	pip3 install -r /app/requirements-base.txt && \
-	pip3 install -r /app/requirements-dev.txt && \
-	pip3 install -r /app/requirements-dashboard.txt && \
+RUN python3 -m pip install -r /app/requirements-heroku.txt && \
+	python3 -m pip install -r /app/requirements-base.txt && \
+	python3 -m pip install -r /app/requirements-dev.txt && \
+	python3 -m pip install -r /app/requirements-dashboard.txt && \
 	rm -rf ~/.cache
 
-RUN pip install -r /app/requirements-heroku.txt && \
-	pip install -r /app/requirements-base.txt && \
-	pip install -r /app/requirements-dev.txt && \
-	pip install -r /app/requirements-dashboard.txt && \
+RUN python -m pip install -r /app/requirements-heroku.txt && \
+	python -m pip install -r /app/requirements-base.txt && \
+	python -m pip install -r /app/requirements-dev.txt && \
+	python -m pip install -r /app/requirements-dashboard.txt && \
 	rm -rf ~/.cache
 
 RUN /pypy/bin/pip install -r /app/requirements-heroku.txt && \

Makefile

@@ -56,7 +56,7 @@ dashboard:
 	python mrq/dashboard/app.py
 
 stack:
-	mongod --smallfiles --noprealloc --nojournal &
+	mongod &
 	redis-server &
 	python mrq/dashboard/app.py &
 

mrq/processes.py

@@ -37,10 +37,10 @@ def request_shutdown_graceful():
                 self.shutdown_graceful()
 
         # First time CTRL-C, try to shutdown gracefully
-        gevent.signal(signal.SIGINT, request_shutdown_graceful)
+        gevent.signal_handler(signal.SIGINT, request_shutdown_graceful)
 
         # User (or Heroku) requests a stop now, just mark tasks as interrupted.
-        gevent.signal(signal.SIGTERM, request_shutdown_now)
+        gevent.signal_handler(signal.SIGTERM, request_shutdown_now)
 
 
 class ProcessPool(object):

mrq/worker.py

@@ -347,7 +347,7 @@ def report_worker(self, w=0):
 
         if self.config["report_file"]:
             with open(self.config["report_file"], "wb") as f:
-                f.write(bytes(json.dumps(report, ensure_ascii=False), 'utf-8'))  # pylint: disable=no-member
+                f.write(bytes(json.dumps(report, ensure_ascii=False, default=str), 'utf-8'))  # pylint: disable=no-member
 
         if "_id" in report:
             del report["_id"]

requirements-base.txt

@@ -4,7 +4,7 @@ pymongo==3.7.2
 gevent>=1.2.2
 ujson>=1.33
 hiredis>=0.1.5
-psutil>=5.1.2,<6.0
+psutil>=5.1.2,<=5.6.3  # https://github.com/giampaolo/psutil/issues/1659#issuecomment-586032229
 objgraph>=1.8.1
 termcolor>=1.1.0
 subprocess32>=3.2.7; python_version < '3.2' and sys.platform != "win32"

requirements-dev.txt

@@ -3,7 +3,7 @@ pylint==1.7.1
 pytest-cov==2.4.0
 pytest-instafail==0.3.0
 pytest-html==1.14.2
-pytest-httpbin==0.2.0
+pytest-httpbin==1.0.0
 pytest-timeout==1.2.0
 
 subprocess32>=3.2.7; python_version < '3.2'

setup.py

@@ -61,7 +61,6 @@ def get_version():
     classifiers=[
         "Programming Language :: Python",
         "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",

tests/conftest.py

@@ -373,15 +373,15 @@ def httpstatic(request):
 
 @pytest.fixture(scope="function")
 def mongodb(request):
-    cmd = "mongod --smallfiles --noprealloc --nojournal"
+    cmd = "mongod --nojournal"
     if os.environ.get("STACK_STARTED"):
         cmd = "sleep 1h"
     return MongoFixture(request, cmd, wait_port=27017, quiet=True)
 
 
 @pytest.fixture(scope="function")
 def mongodb_with_journal(request):
-    cmd = "mongod --smallfiles --noprealloc"
+    cmd = "mongod"
     if os.environ.get("STACK_STARTED"):
         cmd = "sleep 1h"
     return MongoFixture(request, cmd, wait_port=27017, quiet=True)