Job List Limit

changes/8070.feature

@@ -0,0 +1,2 @@
+job_list action now only returns 200 jobs by default.
+Use `limit` or `ckan.jobs.default_list_limit` config option now.

ckan/cli/jobs.py

@@ -40,23 +40,35 @@ def worker(burst: bool, queues: list[str]):
 
 
 @jobs.command(name=u"list", short_help=u"List jobs.")
+@click.option("-l", "--limit", type=click.INT,
+              help="Number of jobs to return. Default: %s" %
+              bg_jobs.DEFAULT_JOB_LIST_LIMIT,
+              default=bg_jobs.DEFAULT_JOB_LIST_LIMIT)
+@click.option("-i", "--ids", is_flag=True, type=click.BOOL,
+              help="Only return a list of job ids.", default=False)
 @click.argument(u"queues", nargs=-1)
-def list_jobs(queues: list[str]):
+def list_jobs(queues: list[str],
+              limit: int = bg_jobs.DEFAULT_JOB_LIST_LIMIT, ids: bool = False):
     """List currently enqueued jobs from the given queues. If no queue
     names are given then the jobs from all queues are listed.
     """
     data_dict = {
         u"queues": list(queues),
+        "limit": limit,
+        "ids_only": ids,
     }
     jobs = logic.get_action(u"job_list")({u"ignore_auth": True}, data_dict)
     if not jobs:
         return click.secho(u"There are no pending jobs.", fg=u"green")
     for job in jobs:
-        if job[u"title"] is None:
-            job[u"title"] = u""
+        if ids:
+            click.secho(job)
         else:
-            job[u"title"] = u'"{}"'.format(job[u"title"])
-        click.secho(u"{created} {id} {queue} {title}".format(**job))
+            if job[u"title"] is None:
+                job[u"title"] = u""
+            else:
+                job[u"title"] = u'"{}"'.format(job[u"title"])
+            click.secho(u"{created} {id} {queue} {title}".format(**job))
 
 
 @jobs.command(short_help=u"Show details about a specific job.")

ckan/lib/jobs.py

@@ -38,6 +38,7 @@
 log = logging.getLogger(__name__)
 
 DEFAULT_QUEUE_NAME = u'default'
+DEFAULT_JOB_LIST_LIMIT = 200
 
 # RQ job queues. Do not use this directly, use ``get_queue`` instead.
 _queues: dict[str, rq.Queue] = {}

ckan/logic/action/get.py

@@ -3160,22 +3160,41 @@ def job_list(context: Context, data_dict: DataDict) -> ActionResult.JobList:
     :param list queues: Queues to list jobs from. If not given then the
         jobs from all queues are listed.
 
+    :param int limit: Number to limit the list of jobs by.
+
+    :param bool ids_only: Whether to return only a list if job IDs or not.
+
     :returns: The currently enqueued background jobs.
     :rtype: list
 
+    Will return the list in the way that RQ workers will execute the jobs.
+    Thus the left most non-empty queue will be emptied first
+    before the next right non-empty one.
+
     .. versionadded:: 2.7
     '''
     _check_access(u'job_list', context, data_dict)
-    dictized_jobs: ActionResult.JobList = []
+    jobs_list: ActionResult.JobList = []
     queues: Any = data_dict.get(u'queues')
+    limit = data_dict.get('limit', config.get('ckan.jobs.default_list_limit',
+                                              jobs.DEFAULT_JOB_LIST_LIMIT))
+    ids_only = asbool(data_dict.get('ids_only', False))
     if queues:
         queues = [jobs.get_queue(q) for q in queues]
     else:
         queues = jobs.get_all_queues()
     for queue in queues:
-        for job in queue.jobs:
-            dictized_jobs.append(jobs.dictize_job(job))
-    return dictized_jobs
+        if ids_only:
+            jobs_list += queue.job_ids[:limit]
+        else:
+            # type_ignore_reason: pyright does not know return from queue.jobs
+            jobs_list += [jobs.dictize_job(j)  # type: ignore
+                          for j in queue.jobs[:limit]]
+        if limit > 0 and len(jobs_list) > limit:
+            # we need to return here if there is a limit
+            # and it has been surpassed
+            return jobs_list[:limit]
+    return jobs_list
 
 
 def job_show(context: Context, data_dict: DataDict) -> ActionResult.JobShow:

ckan/logic/schema.py

@@ -782,10 +782,13 @@ def update_configuration_schema():
 
 @validator_args
 def job_list_schema(
-        ignore_missing: Validator, list_of_strings: Validator
+        ignore_missing: Validator, list_of_strings: Validator,
+        int_validator: Validator, boolean_validator: Validator
 ) -> Schema:
     return {
         u'queues': [ignore_missing, list_of_strings],
+        'limit': [ignore_missing, int_validator],
+        'ids_only': [ignore_missing, boolean_validator],
     }
 
 

ckan/types/logic/action_result.py

@@ -75,7 +75,7 @@
 HelpShow = Optional[str]
 ConfigOptionShow = Any
 ConfigOptionList = List[str]
-JobList = List[AnyDict]
+JobList = Union[List[AnyDict], List[str]]
 JobShow = AnyDict
 ApiTokenList = List[AnyDict]