Merge branch 'tume-master'

* tume-master:
  Use list
  Use newstr from future
  Remove extra list call when iterating
  Experimental python3 support

Author: sylvinus

Makefile

@@ -4,6 +4,9 @@ docker:
 test: docker
 	sh -c "docker run --rm -i -t -p 27017:27017 -p 6379:6379 -p 5555:5555 -p 20020:20020 -v `pwd`:/app:rw -w /app mrq_local python -m pytest tests/ -v --instafail"
 
+test3: docker
+	sh -c "docker run --rm -i -t -p 27017:27017 -p 6379:6379 -p 5555:5555 -p 20020:20020 -v `pwd`:/app:rw -w /app mrq_local python3 -m pytest tests/ -v --instafail"
+
 shell:
 	sh -c "docker run --rm -i -t -p 27017:27017 -p 6379:6379 -p 5555:5555 -p 20020:20020 -p 8000:8000 -v `pwd`:/app:rw -w /app mrq_local bash"
 

examples/simple_crawler/crawler.py

@@ -86,16 +86,16 @@ def run(self, params):
 
         collection = connections.mongodb_jobs.simple_crawler_urls
 
-        print
-        print "Crawl stats"
-        print "==========="
-        print "URLs queued: %s" % collection.find().count()
-        print "URLs successfully crawled: %s" % collection.find({"fetched_date": {"$exists": True}}).count()
-        print "URLs redirected: %s" % collection.find({"redirected_to": {"$exists": True}}).count()
-        print "Bytes fetched: %s" % (list(collection.aggregate(
+        print()
+        print( "Crawl stats")
+        print( "===========")
+        print( "URLs queued: %s" % collection.find().count())
+        print( "URLs successfully crawled: %s" % collection.find({"fetched_date": {"$exists": True}}).count())
+        print( "URLs redirected: %s" % collection.find({"redirected_to": {"$exists": True}}).count())
+        print( "Bytes fetched: %s" % (list(collection.aggregate(
             {"$group": {"_id": None, "sum": {"$sum": "$html_length"}}}
-        )) or [{}])[0].get("sum", 0)
-        print
+        )) or [{}])[0].get("sum", 0))
+        print()
 
 
 class Reset(Task):

mrq/basetasks/cleaning.py

@@ -1,3 +1,4 @@
+from builtins import str
 from mrq.queue import Queue
 from mrq.task import Task
 from mrq.job import Job

mrq/basetasks/utils.py

@@ -1,3 +1,6 @@
+from __future__ import print_function
+from future.utils import itervalues
+from builtins import str
 from mrq.task import Task
 from mrq.queue import Queue
 from bson import ObjectId
@@ -50,7 +53,7 @@ def build_query(self):
         if self.params.get("params"):
             params_dict = json.loads(self.params.get("params"))  # pylint: disable=no-member
 
-            for key in params_dict.keys():
+            for key in params_dict:
                 query["params.%s" % key] = params_dict[key]
 
         return query
@@ -76,7 +79,7 @@ def perform_action(self, action, query, destination_queue):
             else:
 
                 tasks_defs = get_current_config().get("tasks", {})
-                tasks_ttls = [cfg.get("result_ttl", 0) for cfg in tasks_defs.values()]
+                tasks_ttls = [cfg.get("result_ttl", 0) for cfg in itervalues(tasks_defs)]
 
                 result_ttl = max([default_job_timeout] + tasks_ttls)
 
@@ -92,7 +95,7 @@ def perform_action(self, action, query, destination_queue):
             # In this case we could also loose some jobs that were queued after
             # the MongoDB update. They will be "lost" and requeued later like the other case
             # after the Redis BLPOP
-            if query.keys() == ["queue"]:
+            if list(query.keys()) == ["queue"]:
                 Queue(query["queue"]).empty()
 
         elif action in ("requeue", "requeue_retry"):
@@ -135,6 +138,6 @@ def perform_action(self, action, query, destination_queue):
                     Queue(destination_queue or queue, add_to_known_queues=True).enqueue_job_ids(
                         [str(x) for x in jobs_by_queue[queue]])
 
-        print stats
+        print(stats)
 
         return stats

mrq/bin/mrq_run.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
 import os
 
 # Needed to make getaddrinfo() work in pymongo on Mac OS X
@@ -40,13 +42,13 @@ def main():
         # mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"}
         for group in utils.group_iter(cfg["taskargs"], n=2):
             if len(group) != 2:
-                print "Number of arguments wasn't even"
+                print("Number of arguments wasn't even")
                 sys.exit(1)
             params[group[0]] = group[1]
 
     if cfg["queue"]:
         ret = queue_job(cfg["taskpath"], params, queue=cfg["queue"])
-        print ret
+        print(ret)
     else:
         worker_class = load_class_by_path(cfg["worker_class"])
         job = worker_class.job_class(None)
@@ -58,7 +60,7 @@ def main():
         job.datestarted = datetime.datetime.utcnow()
         set_current_job(job)
         ret = job.perform()
-        print json_stdlib.dumps(ret, cls=MongoJSONEncoder)  # pylint: disable=no-member
+        print(json_stdlib.dumps(ret, cls=MongoJSONEncoder))  # pylint: disable=no-member
 
     # This shouldn't be needed as the process will exit and close any remaining sockets
     # connections.redis.connection_pool.disconnect()

mrq/bin/mrq_worker.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import os
+from builtins import str
 
 # Needed to make getaddrinfo() work in pymongo on Mac OS X
 # Docs mention it's a better choice for Linux as well.
@@ -8,15 +9,19 @@
     os.environ["GEVENT_RESOLVER"] = "ares"
 
 from gevent import monkey
-monkey.patch_all()
+monkey.patch_all(subprocess=False)
 
 import sys
 import tempfile
 import signal
-import subprocess32 as subprocess
 import psutil
 import argparse
 
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
+
 sys.path.insert(0, os.getcwd())
 
 from mrq import config

mrq/config.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+from builtins import str
 import argparse
 import os
 import sys
@@ -325,7 +327,7 @@ def add_parser_args(parser, config_type):
             '--report_file',
             default="",
             action='store',
-            type=unicode,
+            type=str,
             help='Filepath of a json dump of the worker status. Disabled if none')
 
         parser.add_argument(
@@ -422,7 +424,7 @@ def get_config(
     # line
     from_args = {}
     if "args" in sources:
-        for k, v in parser.parse_args().__dict__.iteritems():
+        for k, v in parser.parse_args().__dict__.items():
             if default_config[k] != v:
                 from_args[k] = v
 
@@ -443,7 +445,7 @@ def get_config(
         sys.path.insert(0, os.path.dirname(config_file))
         config_module = __import__(os.path.basename(config_file.replace(".py", "")))
         sys.path.pop(0)
-        for k, v in config_module.__dict__.iteritems():
+        for k, v in config_module.__dict__.items():
 
             # We only keep variables starting with an uppercase character.
             if k[0].isupper():
@@ -452,7 +454,7 @@ def get_config(
     # Merge the config in the order given by the user
     merged_config = default_config
 
-    config_keys = set(default_config.keys() + from_file.keys())
+    config_keys = set(list(default_config.keys()) + list(from_file.keys()))
 
     for part in sources:
         for name in config_keys:
@@ -484,11 +486,11 @@ def print_profiling():
         atexit.register(print_profiling)
 
     if merged_config["version"]:
-        print "MRQ version: %s" % VERSION
-        print "Python version: %s" % sys.version
+        print("MRQ version: %s" % VERSION)
+        print("Python version: %s" % sys.version)
         sys.exit(1)
 
     if "no_import_patch" in from_args:
-        print "WARNING: --no_import_patch will be deprecated in MRQ 1.0!"
+        print("WARNING: --no_import_patch will be deprecated in MRQ 1.0!")
 
     return merged_config

mrq/context.py

@@ -1,7 +1,12 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import next
+from builtins import map
+from past.builtins import basestring
 from .logger import Logger
 import gevent
 import gevent.pool
-import urlparse
+import urllib.parse
 import time
 import pymongo
 import traceback
@@ -108,12 +113,12 @@ def versiontuple(v):
         return tuple(map(int, (v.split("."))))
 
     if attr.startswith("redis"):
-        if type(config_obj) in [str, unicode]:
+        if isinstance(config_obj, basestring):
 
             import redis as pyredis
 
-            urlparse.uses_netloc.append('redis')
-            redis_url = urlparse.urlparse(config_obj)
+            urllib.parse.uses_netloc.append('redis')
+            redis_url = urllib.parse.urlparse(config_obj)
 
             log.info("%s: Connecting to Redis at %s..." %
                      (attr, redis_url.hostname))
@@ -124,7 +129,8 @@ def versiontuple(v):
                 db=int((redis_url.path or "").replace("/", "") or "0"),
                 password=redis_url.password,
                 max_connections=int(config.get("redis_max_connections")),
-                timeout=int(config.get("redis_timeout"))
+                timeout=int(config.get("redis_timeout")),
+                decode_responses=True
             )
             return pyredis.StrictRedis(connection_pool=redis_pool)
 
@@ -134,7 +140,7 @@ def versiontuple(v):
 
     elif attr.startswith("mongodb"):
 
-        if type(config_obj) in [str, unicode]:
+        if isinstance(config_obj, basestring):
 
             if attr == "mongodb_logs" and config_obj == "1":
                 return connections.mongodb_jobs
@@ -218,7 +224,7 @@ def inner_func(*args):
 
         try:
           ret = func(*args)
-        except Exception, exc:
+        except Exception as exc:
           trace = traceback.format_exc()
           log.error("Error in subpool: %s \n%s" % (exc, trace))
           raise
@@ -271,7 +277,7 @@ def inner_func(*args):
 
     try:
       ret = func(*args)
-    except Exception, exc:
+    except Exception as exc:
       trace = traceback.format_exc()
       log.error("Error in subpool: %s \n%s" % (exc, trace))
       raise

mrq/dashboard/app.py

@@ -1,3 +1,7 @@
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
+from future.utils import iteritems
 from gevent import monkey
 monkey.patch_all()
 
@@ -41,7 +45,7 @@
 @requires_auth
 def root():
     return render_template("index.html", MRQ_CONFIG={
-        k: v for k, v in cfg.items() if k in WHITELISTED_MRQ_CONFIG_KEYS
+        k: v for k, v in iteritems(cfg) if k in WHITELISTED_MRQ_CONFIG_KEYS
     })
 
 
@@ -149,10 +153,10 @@ def build_api_datatables_query(req):
             try:
                 params_dict = json.loads(req.args.get("params"))
 
-                for key in params_dict.keys():
+                for key in params_dict:
                     query["params.%s" % key] = params_dict[key]
             except Exception as e:  # pylint: disable=broad-except
-                print "Error will converting form JSON: %s" % e
+                print("Error will converting form JSON: %s" % e)
 
     return query
 
@@ -313,7 +317,7 @@ def api_job_traceback(job_id):
 @app.route('/api/jobaction', methods=["POST"])
 @requires_auth
 def api_job_action():
-    params = {k: v for k, v in request.form.iteritems()}
+    params = {k: v for k, v in iteritems(request.form)}
     if params.get("status") and "-" in params.get("status"):
         params["status"] = params.get("status").split("-")
     return jsonify({"job_id": queue_job("mrq.basetasks.utils.JobAction",

mrq/job.py

@@ -1,3 +1,7 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import object
 import datetime
 from bson import ObjectId
 from redis.exceptions import LockError
@@ -11,12 +15,12 @@
 from collections import defaultdict
 import traceback
 import sys
-import urlparse
+import urllib.parse
 import re
 import linecache
 import fnmatch
 import encodings
-import copy_reg
+import copyreg
 from . import context
 
 
@@ -55,7 +59,10 @@ def __init__(self, job_id, queue=None, start=False, fetch=False):
         if job_id is None:
             self.id = None
         else:
-            self.id = ObjectId(job_id)
+            if isinstance(job_id, bytes):
+                self.id = ObjectId(job_id.decode('utf-8'))
+            else:
+                self.id = ObjectId(job_id)
 
         self.data = None
         self.saved = True
@@ -516,10 +523,10 @@ def set_current_io(self, io_data):
     def trace_memory_clean_caches(self):
         """ Avoid polluting results with some builtin python caches """
 
-        urlparse.clear_cache()
+        urllib.parse.clear_cache()
         re.purge()
         linecache.clearcache()
-        copy_reg.clear_extension_cache()
+        copyreg.clear_extension_cache()
 
         if hasattr(fnmatch, "purge"):
             fnmatch.purge()  # pylint: disable=no-member