hiderfong
532 lines
15 KiB
Python
532 lines
15 KiB
Python
import logging
|
|
|
|
from tornado import web
|
|
|
|
from . import BaseApiHandler
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ControlHandler(BaseApiHandler):
|
|
def is_worker(self, workername):
|
|
return workername and workername in self.application.workers
|
|
|
|
def error_reason(self, workername, response):
|
|
"extracts error message from response"
|
|
for res in response:
|
|
try:
|
|
return res[workername].get('error', 'Unknown reason')
|
|
except KeyError:
|
|
pass
|
|
logger.error("Failed to extract error reason from '%s'", response)
|
|
return 'Unknown reason'
|
|
|
|
|
|
class WorkerShutDown(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, workername):
|
|
"""
|
|
Shut down a worker
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/worker/shutdown/celery@worker2 HTTP/1.1
|
|
Content-Length: 0
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 29
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "Shutting down!"
|
|
}
|
|
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 404: unknown worker
|
|
"""
|
|
if not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
logger.info("Shutting down '%s' worker", workername)
|
|
self.capp.control.broadcast('shutdown', destination=[workername])
|
|
self.write(dict(message="Shutting down!"))
|
|
|
|
|
|
class WorkerPoolRestart(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, workername):
|
|
"""
|
|
Restart worker's pool
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/worker/pool/restart/celery@worker2 HTTP/1.1
|
|
Content-Length: 0
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 56
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "Restarting 'celery@worker2' worker's pool"
|
|
}
|
|
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 403: pool restart is not enabled (see CELERYD_POOL_RESTARTS)
|
|
:statuscode 404: unknown worker
|
|
"""
|
|
if not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
logger.info("Restarting '%s' worker's pool", workername)
|
|
response = self.capp.control.broadcast(
|
|
'pool_restart', arguments={'reload': False},
|
|
destination=[workername], reply=True)
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=f"Restarting '{workername}' worker's pool"))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(workername, response)
|
|
self.write(f"Failed to restart the '{workername}' pool: {reason}")
|
|
|
|
|
|
class WorkerPoolGrow(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, workername):
|
|
"""
|
|
Grow worker's pool
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/worker/pool/grow/celery@worker2?n=3 HTTP/1.1
|
|
Content-Length: 0
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 58
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "Growing 'celery@worker2' worker's pool by 3"
|
|
}
|
|
|
|
:query n: number of pool processes to grow, default is 1
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 403: failed to grow
|
|
:statuscode 404: unknown worker
|
|
"""
|
|
|
|
if not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
n = self.get_argument('n', default=1, type=int)
|
|
|
|
logger.info("Growing '%s' worker's pool by '%s'", workername, n)
|
|
response = self.capp.control.pool_grow(
|
|
n=n, reply=True, destination=[workername])
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=f"Growing '{workername}' worker's pool by {n}"))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(workername, response)
|
|
self.write(f"Failed to grow '{workername}' worker's pool: {reason}")
|
|
|
|
|
|
class WorkerPoolShrink(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, workername):
|
|
"""
|
|
Shrink worker's pool
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/worker/pool/shrink/celery@worker2 HTTP/1.1
|
|
Content-Length: 0
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 60
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "Shrinking 'celery@worker2' worker's pool by 1"
|
|
}
|
|
|
|
:query n: number of pool processes to shrink, default is 1
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 403: failed to shrink
|
|
:statuscode 404: unknown worker
|
|
"""
|
|
|
|
if not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
n = self.get_argument('n', default=1, type=int)
|
|
|
|
logger.info("Shrinking '%s' worker's pool by '%s'", workername, n)
|
|
response = self.capp.control.pool_shrink(
|
|
n=n, reply=True, destination=[workername])
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=f"Shrinking '{workername}' worker's pool by {n}"))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(workername, response)
|
|
self.write(f"Failed to shrink '{workername}' worker's pool: {reason}")
|
|
|
|
|
|
class WorkerPoolAutoscale(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, workername):
|
|
"""
|
|
Autoscale worker pool
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/worker/pool/autoscale/celery@worker2?min=3&max=10 HTTP/1.1
|
|
Content-Length: 0
|
|
Content-Type: application/x-www-form-urlencoded; charset=utf-8
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 66
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "Autoscaling 'celery@worker2' worker (min=3, max=10)"
|
|
}
|
|
|
|
:query min: minimum number of pool processes
|
|
:query max: maximum number of pool processes
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 403: autoscaling is not enabled (see CELERYD_AUTOSCALER)
|
|
:statuscode 404: unknown worker
|
|
"""
|
|
|
|
if not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
min = self.get_argument('min', type=int)
|
|
max = self.get_argument('max', type=int)
|
|
|
|
logger.info("Autoscaling '%s' worker by '%s'",
|
|
workername, (min, max))
|
|
response = self.capp.control.broadcast(
|
|
'autoscale', arguments={'min': min, 'max': max},
|
|
destination=[workername], reply=True)
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=f"Autoscaling '{workername}' worker "
|
|
"(min={min}, max={max})"))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(workername, response)
|
|
self.write(f"Failed to autoscale '{workername}' worker: {reason}")
|
|
|
|
|
|
class WorkerQueueAddConsumer(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, workername):
|
|
"""
|
|
Start consuming from a queue
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/worker/queue/add-consumer/celery@worker2?queue=sample-queue
|
|
Content-Length: 0
|
|
Content-Type: application/x-www-form-urlencoded; charset=utf-8
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 40
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "add consumer sample-queue"
|
|
}
|
|
|
|
:query queue: the name of a new queue
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 403: failed to add consumer
|
|
:statuscode 404: unknown worker
|
|
"""
|
|
if not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
queue = self.get_argument('queue')
|
|
|
|
logger.info("Adding consumer '%s' to worker '%s'",
|
|
queue, workername)
|
|
response = self.capp.control.broadcast(
|
|
'add_consumer', arguments={'queue': queue},
|
|
destination=[workername], reply=True)
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=response[0][workername]['ok']))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(workername, response)
|
|
self.write(f"Failed to add '{queue}' consumer to '{workername}' worker: {reason}")
|
|
|
|
|
|
class WorkerQueueCancelConsumer(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, workername):
|
|
"""
|
|
Stop consuming from a queue
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/worker/queue/cancel-consumer/celery@worker2?queue=sample-queue
|
|
Content-Length: 0
|
|
Content-Type: application/x-www-form-urlencoded; charset=utf-8
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 52
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "no longer consuming from sample-queue"
|
|
}
|
|
|
|
:query queue: the name of queue
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 403: failed to cancel consumer
|
|
:statuscode 404: unknown worker
|
|
"""
|
|
if not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
queue = self.get_argument('queue')
|
|
|
|
logger.info("Canceling consumer '%s' from worker '%s'",
|
|
queue, workername)
|
|
response = self.capp.control.broadcast(
|
|
'cancel_consumer', arguments={'queue': queue},
|
|
destination=[workername], reply=True)
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=response[0][workername]['ok']))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(workername, response)
|
|
self.write(f"Failed to cancel '{queue}' consumer from '{workername}' worker: {reason}")
|
|
|
|
|
|
class TaskRevoke(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, taskid):
|
|
"""
|
|
Revoke a task
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/task/revoke/1480b55c-b8b2-462c-985e-24af3e9158f9?terminate=true
|
|
Content-Length: 0
|
|
Content-Type: application/x-www-form-urlencoded; charset=utf-8
|
|
Host: localhost:5555
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 61
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "Revoked '1480b55c-b8b2-462c-985e-24af3e9158f9'"
|
|
}
|
|
|
|
:query terminate: terminate the task if it is running
|
|
:query signal: name of signal to send to process if terminate (default: 'SIGTERM')
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
"""
|
|
logger.info("Revoking task '%s'", taskid)
|
|
terminate = self.get_argument('terminate', default=False, type=bool)
|
|
signal = self.get_argument('signal', default='SIGTERM', type=str)
|
|
self.capp.control.revoke(taskid, terminate=terminate, signal=signal)
|
|
self.write(dict(message=f"Revoked '{taskid}'"))
|
|
|
|
|
|
class TaskTimout(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, taskname):
|
|
"""
|
|
Change soft and hard time limits for a task
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/task/timeout/tasks.sleep HTTP/1.1
|
|
Content-Length: 44
|
|
Content-Type: application/x-www-form-urlencoded; charset=utf-8
|
|
Host: localhost:5555
|
|
|
|
soft=30&hard=100&workername=celery%40worker1
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 46
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "time limits set successfully"
|
|
}
|
|
|
|
:query workername: worker name
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 404: unknown task/worker
|
|
"""
|
|
workername = self.get_argument('workername')
|
|
hard = self.get_argument('hard', default=None, type=float)
|
|
soft = self.get_argument('soft', default=None, type=float)
|
|
|
|
if taskname not in self.capp.tasks:
|
|
raise web.HTTPError(404, f"Unknown task '{taskname}'")
|
|
if workername is not None and not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
logger.info("Setting timeouts for '%s' task (%s, %s)",
|
|
taskname, soft, hard)
|
|
destination = [workername] if workername is not None else None
|
|
response = self.capp.control.time_limit(
|
|
taskname, reply=True, hard=hard, soft=soft,
|
|
destination=destination)
|
|
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=response[0][workername]['ok']))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(taskname, response)
|
|
self.write(f"Failed to set timeouts: '{reason}'")
|
|
|
|
|
|
class TaskRateLimit(ControlHandler):
|
|
@web.authenticated
|
|
def post(self, taskname):
|
|
"""
|
|
Change rate limit for a task
|
|
|
|
**Example request**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
POST /api/task/rate-limit/tasks.sleep HTTP/1.1
|
|
Content-Length: 41
|
|
Content-Type: application/x-www-form-urlencoded; charset=utf-8
|
|
Host: localhost:5555
|
|
|
|
ratelimit=200&workername=celery%40worker1
|
|
|
|
**Example response**:
|
|
|
|
.. sourcecode:: http
|
|
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 61
|
|
Content-Type: application/json; charset=UTF-8
|
|
|
|
{
|
|
"message": "new rate limit set successfully"
|
|
}
|
|
|
|
:query workername: worker name
|
|
:reqheader Authorization: optional OAuth token to authenticate
|
|
:statuscode 200: no error
|
|
:statuscode 401: unauthorized request
|
|
:statuscode 404: unknown task/worker
|
|
"""
|
|
workername = self.get_argument('workername')
|
|
ratelimit = self.get_argument('ratelimit')
|
|
|
|
if taskname not in self.capp.tasks:
|
|
raise web.HTTPError(404, f"Unknown task '{taskname}'")
|
|
if workername is not None and not self.is_worker(workername):
|
|
raise web.HTTPError(404, f"Unknown worker '{workername}'")
|
|
|
|
logger.info("Setting '%s' rate limit for '%s' task",
|
|
ratelimit, taskname)
|
|
destination = [workername] if workername is not None else None
|
|
response = self.capp.control.rate_limit(
|
|
taskname, ratelimit, reply=True, destination=destination)
|
|
if response and 'ok' in response[0][workername]:
|
|
self.write(dict(message=response[0][workername]['ok']))
|
|
else:
|
|
logger.error(response)
|
|
self.set_status(403)
|
|
reason = self.error_reason(taskname, response)
|
|
self.write(f"Failed to set rate limit: '{reason}'")
|