feat: initial commit - Phase 1 & 2 core features

This commit is contained in:
hiderfong
2026-04-22 17:07:33 +08:00
commit 1773bda06b
25005 changed files with 6252106 additions and 0 deletions
@@ -0,0 +1,2 @@
VERSION = (2, 0, 1)
__version__ = '.'.join(map(str, VERSION))
@@ -0,0 +1,12 @@
import sys
from celery.bin.celery import main as _main, celery
from flower.command import flower
def main():
celery.add_command(flower)
sys.exit(_main())
if __name__ == "__main__":
main()
@@ -0,0 +1,23 @@
import os
import tornado.web
from ..utils import strtobool
from ..views import BaseHandler
class BaseApiHandler(BaseHandler):
def prepare(self):
enable_api = strtobool(os.environ.get(
'FLOWER_UNAUTHENTICATED_API') or "false")
if not (self.application.options.basic_auth or self.application.options.auth) and not enable_api:
raise tornado.web.HTTPError(
401, "FLOWER_UNAUTHENTICATED_API environment variable is required to enable API without authentication")
def write_error(self, status_code, **kwargs):
exc_info = kwargs.get('exc_info')
log_message = exc_info[1].log_message
if log_message:
self.write(log_message)
self.set_status(status_code)
self.finish()
@@ -0,0 +1,531 @@
import logging
from tornado import web
from . import BaseApiHandler
logger = logging.getLogger(__name__)
class ControlHandler(BaseApiHandler):
def is_worker(self, workername):
return workername and workername in self.application.workers
def error_reason(self, workername, response):
"extracts error message from response"
for res in response:
try:
return res[workername].get('error', 'Unknown reason')
except KeyError:
pass
logger.error("Failed to extract error reason from '%s'", response)
return 'Unknown reason'
class WorkerShutDown(ControlHandler):
@web.authenticated
def post(self, workername):
"""
Shut down a worker
**Example request**:
.. sourcecode:: http
POST /api/worker/shutdown/celery@worker2 HTTP/1.1
Content-Length: 0
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 29
Content-Type: application/json; charset=UTF-8
{
"message": "Shutting down!"
}
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 404: unknown worker
"""
if not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
logger.info("Shutting down '%s' worker", workername)
self.capp.control.broadcast('shutdown', destination=[workername])
self.write(dict(message="Shutting down!"))
class WorkerPoolRestart(ControlHandler):
@web.authenticated
def post(self, workername):
"""
Restart worker's pool
**Example request**:
.. sourcecode:: http
POST /api/worker/pool/restart/celery@worker2 HTTP/1.1
Content-Length: 0
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 56
Content-Type: application/json; charset=UTF-8
{
"message": "Restarting 'celery@worker2' worker's pool"
}
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 403: pool restart is not enabled (see CELERYD_POOL_RESTARTS)
:statuscode 404: unknown worker
"""
if not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
logger.info("Restarting '%s' worker's pool", workername)
response = self.capp.control.broadcast(
'pool_restart', arguments={'reload': False},
destination=[workername], reply=True)
if response and 'ok' in response[0][workername]:
self.write(dict(message=f"Restarting '{workername}' worker's pool"))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(workername, response)
self.write(f"Failed to restart the '{workername}' pool: {reason}")
class WorkerPoolGrow(ControlHandler):
@web.authenticated
def post(self, workername):
"""
Grow worker's pool
**Example request**:
.. sourcecode:: http
POST /api/worker/pool/grow/celery@worker2?n=3 HTTP/1.1
Content-Length: 0
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 58
Content-Type: application/json; charset=UTF-8
{
"message": "Growing 'celery@worker2' worker's pool by 3"
}
:query n: number of pool processes to grow, default is 1
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 403: failed to grow
:statuscode 404: unknown worker
"""
if not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
n = self.get_argument('n', default=1, type=int)
logger.info("Growing '%s' worker's pool by '%s'", workername, n)
response = self.capp.control.pool_grow(
n=n, reply=True, destination=[workername])
if response and 'ok' in response[0][workername]:
self.write(dict(message=f"Growing '{workername}' worker's pool by {n}"))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(workername, response)
self.write(f"Failed to grow '{workername}' worker's pool: {reason}")
class WorkerPoolShrink(ControlHandler):
@web.authenticated
def post(self, workername):
"""
Shrink worker's pool
**Example request**:
.. sourcecode:: http
POST /api/worker/pool/shrink/celery@worker2 HTTP/1.1
Content-Length: 0
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 60
Content-Type: application/json; charset=UTF-8
{
"message": "Shrinking 'celery@worker2' worker's pool by 1"
}
:query n: number of pool processes to shrink, default is 1
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 403: failed to shrink
:statuscode 404: unknown worker
"""
if not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
n = self.get_argument('n', default=1, type=int)
logger.info("Shrinking '%s' worker's pool by '%s'", workername, n)
response = self.capp.control.pool_shrink(
n=n, reply=True, destination=[workername])
if response and 'ok' in response[0][workername]:
self.write(dict(message=f"Shrinking '{workername}' worker's pool by {n}"))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(workername, response)
self.write(f"Failed to shrink '{workername}' worker's pool: {reason}")
class WorkerPoolAutoscale(ControlHandler):
@web.authenticated
def post(self, workername):
"""
Autoscale worker pool
**Example request**:
.. sourcecode:: http
POST /api/worker/pool/autoscale/celery@worker2?min=3&max=10 HTTP/1.1
Content-Length: 0
Content-Type: application/x-www-form-urlencoded; charset=utf-8
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 66
Content-Type: application/json; charset=UTF-8
{
"message": "Autoscaling 'celery@worker2' worker (min=3, max=10)"
}
:query min: minimum number of pool processes
:query max: maximum number of pool processes
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 403: autoscaling is not enabled (see CELERYD_AUTOSCALER)
:statuscode 404: unknown worker
"""
if not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
min = self.get_argument('min', type=int)
max = self.get_argument('max', type=int)
logger.info("Autoscaling '%s' worker by '%s'",
workername, (min, max))
response = self.capp.control.broadcast(
'autoscale', arguments={'min': min, 'max': max},
destination=[workername], reply=True)
if response and 'ok' in response[0][workername]:
self.write(dict(message=f"Autoscaling '{workername}' worker "
"(min={min}, max={max})"))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(workername, response)
self.write(f"Failed to autoscale '{workername}' worker: {reason}")
class WorkerQueueAddConsumer(ControlHandler):
@web.authenticated
def post(self, workername):
"""
Start consuming from a queue
**Example request**:
.. sourcecode:: http
POST /api/worker/queue/add-consumer/celery@worker2?queue=sample-queue
Content-Length: 0
Content-Type: application/x-www-form-urlencoded; charset=utf-8
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 40
Content-Type: application/json; charset=UTF-8
{
"message": "add consumer sample-queue"
}
:query queue: the name of a new queue
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 403: failed to add consumer
:statuscode 404: unknown worker
"""
if not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
queue = self.get_argument('queue')
logger.info("Adding consumer '%s' to worker '%s'",
queue, workername)
response = self.capp.control.broadcast(
'add_consumer', arguments={'queue': queue},
destination=[workername], reply=True)
if response and 'ok' in response[0][workername]:
self.write(dict(message=response[0][workername]['ok']))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(workername, response)
self.write(f"Failed to add '{queue}' consumer to '{workername}' worker: {reason}")
class WorkerQueueCancelConsumer(ControlHandler):
@web.authenticated
def post(self, workername):
"""
Stop consuming from a queue
**Example request**:
.. sourcecode:: http
POST /api/worker/queue/cancel-consumer/celery@worker2?queue=sample-queue
Content-Length: 0
Content-Type: application/x-www-form-urlencoded; charset=utf-8
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 52
Content-Type: application/json; charset=UTF-8
{
"message": "no longer consuming from sample-queue"
}
:query queue: the name of queue
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 403: failed to cancel consumer
:statuscode 404: unknown worker
"""
if not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
queue = self.get_argument('queue')
logger.info("Canceling consumer '%s' from worker '%s'",
queue, workername)
response = self.capp.control.broadcast(
'cancel_consumer', arguments={'queue': queue},
destination=[workername], reply=True)
if response and 'ok' in response[0][workername]:
self.write(dict(message=response[0][workername]['ok']))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(workername, response)
self.write(f"Failed to cancel '{queue}' consumer from '{workername}' worker: {reason}")
class TaskRevoke(ControlHandler):
@web.authenticated
def post(self, taskid):
"""
Revoke a task
**Example request**:
.. sourcecode:: http
POST /api/task/revoke/1480b55c-b8b2-462c-985e-24af3e9158f9?terminate=true
Content-Length: 0
Content-Type: application/x-www-form-urlencoded; charset=utf-8
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 61
Content-Type: application/json; charset=UTF-8
{
"message": "Revoked '1480b55c-b8b2-462c-985e-24af3e9158f9'"
}
:query terminate: terminate the task if it is running
:query signal: name of signal to send to process if terminate (default: 'SIGTERM')
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
"""
logger.info("Revoking task '%s'", taskid)
terminate = self.get_argument('terminate', default=False, type=bool)
signal = self.get_argument('signal', default='SIGTERM', type=str)
self.capp.control.revoke(taskid, terminate=terminate, signal=signal)
self.write(dict(message=f"Revoked '{taskid}'"))
class TaskTimout(ControlHandler):
@web.authenticated
def post(self, taskname):
"""
Change soft and hard time limits for a task
**Example request**:
.. sourcecode:: http
POST /api/task/timeout/tasks.sleep HTTP/1.1
Content-Length: 44
Content-Type: application/x-www-form-urlencoded; charset=utf-8
Host: localhost:5555
soft=30&hard=100&workername=celery%40worker1
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 46
Content-Type: application/json; charset=UTF-8
{
"message": "time limits set successfully"
}
:query workername: worker name
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 404: unknown task/worker
"""
workername = self.get_argument('workername')
hard = self.get_argument('hard', default=None, type=float)
soft = self.get_argument('soft', default=None, type=float)
if taskname not in self.capp.tasks:
raise web.HTTPError(404, f"Unknown task '{taskname}'")
if workername is not None and not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
logger.info("Setting timeouts for '%s' task (%s, %s)",
taskname, soft, hard)
destination = [workername] if workername is not None else None
response = self.capp.control.time_limit(
taskname, reply=True, hard=hard, soft=soft,
destination=destination)
if response and 'ok' in response[0][workername]:
self.write(dict(message=response[0][workername]['ok']))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(taskname, response)
self.write(f"Failed to set timeouts: '{reason}'")
class TaskRateLimit(ControlHandler):
@web.authenticated
def post(self, taskname):
"""
Change rate limit for a task
**Example request**:
.. sourcecode:: http
POST /api/task/rate-limit/tasks.sleep HTTP/1.1
Content-Length: 41
Content-Type: application/x-www-form-urlencoded; charset=utf-8
Host: localhost:5555
ratelimit=200&workername=celery%40worker1
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 61
Content-Type: application/json; charset=UTF-8
{
"message": "new rate limit set successfully"
}
:query workername: worker name
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 404: unknown task/worker
"""
workername = self.get_argument('workername')
ratelimit = self.get_argument('ratelimit')
if taskname not in self.capp.tasks:
raise web.HTTPError(404, f"Unknown task '{taskname}'")
if workername is not None and not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
logger.info("Setting '%s' rate limit for '%s' task",
ratelimit, taskname)
destination = [workername] if workername is not None else None
response = self.capp.control.rate_limit(
taskname, ratelimit, reply=True, destination=destination)
if response and 'ok' in response[0][workername]:
self.write(dict(message=response[0][workername]['ok']))
else:
logger.error(response)
self.set_status(403)
reason = self.error_reason(taskname, response)
self.write(f"Failed to set rate limit: '{reason}'")
@@ -0,0 +1,638 @@
import json
import logging
from collections import OrderedDict
from datetime import datetime
from celery import states
from celery.backends.base import DisabledBackend
from celery.contrib.abortable import AbortableAsyncResult
from celery.result import AsyncResult
from tornado import web
from tornado.escape import json_decode
from tornado.ioloop import IOLoop
from tornado.web import HTTPError
from ..utils import tasks
from ..utils.broker import Broker
from . import BaseApiHandler
logger = logging.getLogger(__name__)
class BaseTaskHandler(BaseApiHandler):
DATE_FORMAT = '%Y-%m-%d %H:%M:%S.%f'
def get_task_args(self):
try:
body = self.request.body
options = json_decode(body) if body else {}
except ValueError as e:
raise HTTPError(400, str(e)) from e
if not isinstance(options, dict):
raise HTTPError(400, 'invalid options')
args = options.pop('args', [])
kwargs = options.pop('kwargs', {})
if not isinstance(args, (list, tuple)):
raise HTTPError(400, 'args must be an array')
return args, kwargs, options
@staticmethod
def backend_configured(result):
return not isinstance(result.backend, DisabledBackend)
def write_error(self, status_code, **kwargs):
self.set_status(status_code)
def update_response_result(self, response, result):
if result.state == states.FAILURE:
response.update({'result': self.safe_result(result.result),
'traceback': result.traceback})
else:
response.update({'result': self.safe_result(result.result)})
def normalize_options(self, options):
if 'eta' in options:
options['eta'] = datetime.strptime(options['eta'],
self.DATE_FORMAT)
if 'countdown' in options:
options['countdown'] = float(options['countdown'])
if 'expires' in options:
expires = options['expires']
try:
expires = float(expires)
except ValueError:
expires = datetime.strptime(expires, self.DATE_FORMAT)
options['expires'] = expires
def safe_result(self, result):
"returns json encodable result"
try:
json.dumps(result)
except TypeError:
return repr(result)
return result
class TaskApply(BaseTaskHandler):
@web.authenticated
async def post(self, taskname):
"""
Execute a task by name and wait results
**Example request**:
.. sourcecode:: http
POST /api/task/apply/tasks.add HTTP/1.1
Accept: application/json
Accept-Encoding: gzip, deflate, compress
Content-Length: 16
Content-Type: application/json; charset=utf-8
Host: localhost:5555
{
"args": [1, 2]
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 71
Content-Type: application/json; charset=UTF-8
{
"state": "SUCCESS",
"task-id": "c60be250-fe52-48df-befb-ac66174076e6",
"result": 3
}
:query args: a list of arguments
:query kwargs: a dictionary of arguments
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 404: unknown task
"""
args, kwargs, options = self.get_task_args()
logger.debug("Invoking a task '%s' with '%s' and '%s'",
taskname, args, kwargs)
try:
task = self.capp.tasks[taskname]
except KeyError as exc:
raise HTTPError(404, f"Unknown task '{taskname}'") from exc
try:
self.normalize_options(options)
except ValueError as exc:
raise HTTPError(400, 'Invalid option') from exc
result = task.apply_async(args=args, kwargs=kwargs, **options)
response = {'task-id': result.task_id}
response = await IOLoop.current().run_in_executor(
None, self.wait_results, result, response)
self.write(response)
def wait_results(self, result, response):
# Wait until task finished and do not raise anything
result.get(propagate=False)
# Write results and finish async function
self.update_response_result(response, result)
if self.backend_configured(result):
response.update(state=result.state)
return response
class TaskAsyncApply(BaseTaskHandler):
@web.authenticated
def post(self, taskname):
"""
Execute a task
**Example request**:
.. sourcecode:: http
POST /api/task/async-apply/tasks.add HTTP/1.1
Accept: application/json
Accept-Encoding: gzip, deflate, compress
Content-Length: 16
Content-Type: application/json; charset=utf-8
Host: localhost:5555
{
"args": [1, 2]
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 71
Content-Type: application/json; charset=UTF-8
Date: Sun, 13 Apr 2014 15:55:00 GMT
{
"state": "PENDING",
"task-id": "abc300c7-2922-4069-97b6-a635cc2ac47c"
}
:query args: a list of arguments
:query kwargs: a dictionary of arguments
:query options: a dictionary of `apply_async` keyword arguments
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 404: unknown task
"""
args, kwargs, options = self.get_task_args()
logger.debug("Invoking a task '%s' with '%s' and '%s'",
taskname, args, kwargs)
try:
task = self.capp.tasks[taskname]
except KeyError as exc:
raise HTTPError(404, f"Unknown task '{taskname}'") from exc
try:
self.normalize_options(options)
except ValueError as exc:
raise HTTPError(400, 'Invalid option') from exc
result = task.apply_async(args=args, kwargs=kwargs, **options)
response = {'task-id': result.task_id}
if self.backend_configured(result):
response.update(state=result.state)
self.write(response)
class TaskSend(BaseTaskHandler):
@web.authenticated
def post(self, taskname):
"""
Execute a task by name (doesn't require task sources)
**Example request**:
.. sourcecode:: http
POST /api/task/send-task/tasks.add HTTP/1.1
Accept: application/json
Accept-Encoding: gzip, deflate, compress
Content-Length: 16
Content-Type: application/json; charset=utf-8
Host: localhost:5555
{
"args": [1, 2]
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 71
Content-Type: application/json; charset=UTF-8
{
"state": "SUCCESS",
"task-id": "c60be250-fe52-48df-befb-ac66174076e6"
}
:query args: a list of arguments
:query kwargs: a dictionary of arguments
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 404: unknown task
"""
args, kwargs, options = self.get_task_args()
logger.debug("Invoking task '%s' with '%s' and '%s'",
taskname, args, kwargs)
result = self.capp.send_task(
taskname, args=args, kwargs=kwargs, **options)
response = {'task-id': result.task_id}
if self.backend_configured(result):
response.update(state=result.state)
self.write(response)
class TaskResult(BaseTaskHandler):
@web.authenticated
def get(self, taskid):
"""
Get a task result
**Example request**:
.. sourcecode:: http
GET /api/task/result/c60be250-fe52-48df-befb-ac66174076e6 HTTP/1.1
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 84
Content-Type: application/json; charset=UTF-8
{
"result": 3,
"state": "SUCCESS",
"task-id": "c60be250-fe52-48df-befb-ac66174076e6"
}
:query timeout: how long to wait, in seconds, before the operation times out
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 503: result backend is not configured
"""
timeout = self.get_argument('timeout', None)
timeout = float(timeout) if timeout is not None else None
result = AsyncResult(taskid)
if not self.backend_configured(result):
raise HTTPError(503)
response = {'task-id': taskid, 'state': result.state}
if timeout:
result.get(timeout=timeout, propagate=False)
self.update_response_result(response, result)
elif result.ready():
self.update_response_result(response, result)
self.write(response)
class TaskAbort(BaseTaskHandler):
@web.authenticated
def post(self, taskid):
"""
Abort a running task
**Example request**:
.. sourcecode:: http
POST /api/task/abort/c60be250-fe52-48df-befb-ac66174076e6 HTTP/1.1
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 61
Content-Type: application/json; charset=UTF-8
{
"message": "Aborted '1480b55c-b8b2-462c-985e-24af3e9158f9'"
}
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 503: result backend is not configured
"""
logger.info("Aborting task '%s'", taskid)
result = AbortableAsyncResult(taskid)
if not self.backend_configured(result):
raise HTTPError(503)
result.abort()
self.write(dict(message=f"Aborted '{taskid}'"))
class GetQueueLengths(BaseTaskHandler):
@web.authenticated
async def get(self):
"""
Return length of all active queues
**Example request**:
.. sourcecode:: http
GET /api/queues/length
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 94
Content-Type: application/json; charset=UTF-8
{
"active_queues": [
{"name": "celery", "messages": 0},
{"name": "video-queue", "messages": 5}
]
}
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 503: result backend is not configured
"""
app = self.application
http_api = None
if app.transport == 'amqp' and app.options.broker_api:
http_api = app.options.broker_api
broker = Broker(app.capp.connection().as_uri(include_password=True),
http_api=http_api, broker_options=self.capp.conf.broker_transport_options,
broker_use_ssl=self.capp.conf.broker_use_ssl)
queues = await broker.queues(self.get_active_queue_names())
self.write({'active_queues': queues})
class ListTasks(BaseTaskHandler):
@web.authenticated
def get(self):
"""
List tasks
**Example request**:
.. sourcecode:: http
GET /api/tasks HTTP/1.1
Host: localhost:5555
User-Agent: HTTPie/0.8.0
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 1109
Content-Type: application/json; charset=UTF-8
Etag: "b2478118015c8b825f7b88ce6b660e5449746c37"
Server: TornadoServer/3.1.1
{
"e42ceb2d-8730-47b5-8b4d-8e0d2a1ef7c9": {
"args": "[3, 4]",
"client": null,
"clock": 1079,
"eta": null,
"exception": null,
"exchange": null,
"expires": null,
"failed": null,
"kwargs": "{}",
"name": "tasks.add",
"received": 1398505411.107885,
"result": "'7'",
"retried": null,
"retries": 0,
"revoked": null,
"routing_key": null,
"runtime": 0.01610181899741292,
"sent": null,
"started": 1398505411.108985,
"state": "SUCCESS",
"succeeded": 1398505411.124802,
"timestamp": 1398505411.124802,
"traceback": null,
"uuid": "e42ceb2d-8730-47b5-8b4d-8e0d2a1ef7c9",
"worker": "celery@worker1"
},
"f67ea225-ae9e-42a8-90b0-5de0b24507e0": {
"args": "[1, 2]",
"client": null,
"clock": 1042,
"eta": null,
"exception": null,
"exchange": null,
"expires": null,
"failed": null,
"kwargs": "{}",
"name": "tasks.add",
"received": 1398505395.327208,
"result": "'3'",
"retried": null,
"retries": 0,
"revoked": null,
"routing_key": null,
"runtime": 0.012884548006695695,
"sent": null,
"started": 1398505395.3289,
"state": "SUCCESS",
"succeeded": 1398505395.341089,
"timestamp": 1398505395.341089,
"traceback": null,
"uuid": "f67ea225-ae9e-42a8-90b0-5de0b24507e0",
"worker": "celery@worker1"
}
}
:query limit: maximum number of tasks
:query offset: skip first n tasks
:query sort_by: sort tasks by attribute (name, state, received, started)
:query workername: filter task by workername
:query taskname: filter tasks by taskname
:query state: filter tasks by state
:query received_start: filter tasks by received date (must be greater than) format %Y-%m-%d %H:%M
:query received_end: filter tasks by received date (must be less than) format %Y-%m-%d %H:%M
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
"""
app = self.application
limit = self.get_argument('limit', None)
offset = self.get_argument('offset', default=0, type=int)
worker = self.get_argument('workername', None)
type = self.get_argument('taskname', None)
state = self.get_argument('state', None)
received_start = self.get_argument('received_start', None)
received_end = self.get_argument('received_end', None)
sort_by = self.get_argument('sort_by', None)
search = self.get_argument('search', None)
limit = limit and int(limit)
offset = max(offset, 0)
worker = worker if worker != 'All' else None
type = type if type != 'All' else None
state = state if state != 'All' else None
result = []
for task_id, task in tasks.iter_tasks(
app.events, limit=limit, offset=offset, sort_by=sort_by, type=type,
worker=worker, state=state,
received_start=received_start,
received_end=received_end,
search=search
):
task = tasks.as_dict(task)
worker = task.pop('worker', None)
if worker is not None:
task['worker'] = worker.hostname
result.append((task_id, task))
self.write(OrderedDict(result))
class ListTaskTypes(BaseTaskHandler):
@web.authenticated
def get(self):
"""
List (seen) task types
**Example request**:
.. sourcecode:: http
GET /api/task/types HTTP/1.1
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 44
Content-Type: application/json; charset=UTF-8
{
"task-types": [
"tasks.add",
"tasks.sleep"
]
}
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
"""
seen_task_types = self.application.events.state.task_types()
response = {}
response['task-types'] = seen_task_types
self.write(response)
class TaskInfo(BaseTaskHandler):
@web.authenticated
def get(self, taskid):
"""
Get a task info
**Example request**:
.. sourcecode:: http
GET /api/task/info/91396550-c228-4111-9da4-9d88cfd5ddc6 HTTP/1.1
Accept: */*
Accept-Encoding: gzip, deflate, compress
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 575
Content-Type: application/json; charset=UTF-8
{
"args": "[2, 2]",
"client": null,
"clock": 25,
"eta": null,
"exception": null,
"exchange": null,
"expires": null,
"failed": null,
"kwargs": "{}",
"name": "tasks.add",
"received": 1400806241.970742,
"result": "'4'",
"retried": null,
"retries": null,
"revoked": null,
"routing_key": null,
"runtime": 2.0037889280356467,
"sent": null,
"started": 1400806241.972624,
"state": "SUCCESS",
"succeeded": 1400806243.975336,
"task-id": "91396550-c228-4111-9da4-9d88cfd5ddc6",
"timestamp": 1400806243.975336,
"traceback": null,
"worker": "celery@worker1"
}
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 404: unknown task
"""
task = tasks.get_task_by_id(self.application.events, taskid)
if not task:
raise HTTPError(404, f"Unknown task '{taskid}'")
response = task.as_dict()
if task.worker is not None:
response['worker'] = task.worker.hostname
self.write(response)
@@ -0,0 +1,183 @@
import asyncio
import logging
from tornado import web
from .control import ControlHandler
logger = logging.getLogger(__name__)
class ListWorkers(ControlHandler):
@web.authenticated
async def get(self):
"""
List workers
**Example request**:
.. sourcecode:: http
GET /api/workers HTTP/1.1
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 1526
Content-Type: application/json; charset=UTF-8
Date: Tue, 28 Jul 2015 01:32:38 GMT
Etag: "fcdd75d85a82b4052275e28871d199aac1ece21c"
Server: TornadoServer/4.0.2
{
"celery@worker1": {
"active_queues": [
{
"alias": null,
"auto_delete": false,
"binding_arguments": null,
"bindings": [],
"durable": true,
"exchange": {
"arguments": null,
"auto_delete": false,
"delivery_mode": 2,
"durable": true,
"name": "celery",
"passive": false,
"type": "direct"
},
"exclusive": false,
"name": "celery",
"no_ack": false,
"queue_arguments": null,
"routing_key": "celery"
}
],
"conf": {
"CELERYBEAT_SCHEDULE": {},
"CELERY_INCLUDE": [
"celery.app.builtins",
"__main__"
],
"CELERY_SEND_TASK_SENT_EVENT": true,
"CELERY_TIMEZONE": "UTC"
},
"registered": [
"tasks.add",
"tasks.echo",
"tasks.error",
"tasks.retry",
"tasks.sleep"
],
"stats": {
"broker": {
"alternates": [],
"connect_timeout": 4,
"heartbeat": null,
"hostname": "127.0.0.1",
"insist": false,
"login_method": "AMQPLAIN",
"port": 5672,
"ssl": false,
"transport": "amqp",
"transport_options": {},
"uri_prefix": null,
"userid": "guest",
"virtual_host": "/"
},
"clock": "918",
"pid": 90494,
"pool": {
"max-concurrency": 4,
"max-tasks-per-child": "N/A",
"processes": [
90499,
90500,
90501,
90502
],
"put-guarded-by-semaphore": false,
"timeouts": [
0,
0
],
"writes": {
"all": "100.00%",
"avg": "100.00%",
"inqueues": {
"active": 0,
"total": 4
},
"raw": "1",
"total": 1
}
},
"prefetch_count": 16,
"rusage": {
"idrss": 0,
"inblock": 211,
"isrss": 0,
"ixrss": 0,
"majflt": 6,
"maxrss": 26996736,
"minflt": 11450,
"msgrcv": 4968,
"msgsnd": 1227,
"nivcsw": 1367,
"nsignals": 0,
"nswap": 0,
"nvcsw": 1855,
"oublock": 93,
"stime": 0.414564,
"utime": 0.975726
},
"total": {
"tasks.add": 1
}
},
"timestamp": 1438049312.073402
}
}
:query refresh: run inspect to get updated list of workers
:query workername: get info for workername
:query status: only get worker status info
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
"""
refresh = self.get_argument('refresh', default=False, type=bool)
status = self.get_argument('status', default=False, type=bool)
workername = self.get_argument('workername', default=None)
if refresh:
try:
await asyncio.wait(self.application.update_workers(workername=workername))
except Exception as e:
msg = f"Failed to update workers: {e}"
logger.error(msg)
raise web.HTTPError(503, msg)
if status:
info = {}
for name, worker in self.application.events.state.workers.items():
info[name] = worker.alive
self.write(info)
return
if self.application.workers and not refresh and\
workername in self.application.workers:
self.write({workername: self.application.workers[workername]})
return
if workername and not self.is_worker(workername):
raise web.HTTPError(404, f"Unknown worker '{workername}'")
if workername:
self.write({workername: self.application.workers[workername]})
else:
self.write(self.application.workers)
@@ -0,0 +1,103 @@
import sys
import logging
from concurrent.futures import ThreadPoolExecutor
import celery
import tornado.web
from tornado import ioloop
from tornado.httpserver import HTTPServer
from tornado.web import url
from .urls import handlers as default_handlers
from .events import Events
from .inspector import Inspector
from .options import default_options
logger = logging.getLogger(__name__)
if sys.version_info[0] == 3 and sys.version_info[1] >= 8 and sys.platform.startswith('win'):
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
# pylint: disable=consider-using-f-string
def rewrite_handler(handler, url_prefix):
if isinstance(handler, url):
return url("/{}{}".format(url_prefix.strip("/"), handler.regex.pattern),
handler.handler_class, handler.kwargs, handler.name)
return ("/{}{}".format(url_prefix.strip("/"), handler[0]), handler[1])
class Flower(tornado.web.Application):
pool_executor_cls = ThreadPoolExecutor
max_workers = None
def __init__(self, options=None, capp=None, events=None,
io_loop=None, **kwargs):
handlers = default_handlers
if options is not None and options.url_prefix:
handlers = [rewrite_handler(h, options.url_prefix) for h in handlers]
kwargs.update(handlers=handlers)
super().__init__(**kwargs)
self.options = options or default_options
self.io_loop = io_loop or ioloop.IOLoop.instance()
self.ssl_options = kwargs.get('ssl_options', None)
self.capp = capp or celery.Celery()
self.capp.loader.import_default_modules()
self.executor = self.pool_executor_cls(max_workers=self.max_workers)
self.io_loop.set_default_executor(self.executor)
self.inspector = Inspector(self.io_loop, self.capp, self.options.inspect_timeout / 1000.0)
self.events = events or Events(
self.capp,
db=self.options.db,
persistent=self.options.persistent,
state_save_interval=self.options.state_save_interval,
enable_events=self.options.enable_events,
io_loop=self.io_loop,
max_workers_in_memory=self.options.max_workers,
max_tasks_in_memory=self.options.max_tasks)
self.started = False
def start(self):
self.events.start()
if not self.options.unix_socket:
self.listen(self.options.port, address=self.options.address,
ssl_options=self.ssl_options,
xheaders=self.options.xheaders)
else:
from tornado.netutil import bind_unix_socket
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket, mode=0o777)
server.add_socket(socket)
self.started = True
self.update_workers()
self.io_loop.start()
def stop(self):
if self.started:
self.events.stop()
logging.debug("Stopping executors...")
self.executor.shutdown(wait=False)
logging.debug("Stopping event loop...")
self.io_loop.stop()
self.started = False
@property
def transport(self):
return getattr(self.capp.connection().transport, 'driver_type', None)
@property
def workers(self):
return self.inspector.workers
def update_workers(self, workername=None):
return self.inspector.inspect(workername)
@@ -0,0 +1,181 @@
import os
import sys
import atexit
import signal
import logging
from pprint import pformat
from logging import NullHandler
import click
from tornado.options import options
from tornado.options import parse_command_line, parse_config_file
from tornado.log import enable_pretty_logging
from celery.bin.base import CeleryCommand
from .app import Flower
from .urls import settings
from .utils import abs_path, prepend_url, strtobool
from .options import DEFAULT_CONFIG_FILE, default_options
from .views.auth import validate_auth_option
logger = logging.getLogger(__name__)
ENV_VAR_PREFIX = 'FLOWER_'
def sigterm_handler(signum, _):
logger.info('%s detected, shutting down', signum)
sys.exit(0)
@click.command(cls=CeleryCommand,
context_settings={
'ignore_unknown_options': True
})
@click.argument("tornado_argv", nargs=-1, type=click.UNPROCESSED)
@click.pass_context
def flower(ctx, tornado_argv):
"""Web based tool for monitoring and administrating Celery clusters."""
warn_about_celery_args_used_in_flower_command(ctx, tornado_argv)
apply_env_options()
apply_options(sys.argv[0], tornado_argv)
extract_settings()
setup_logging()
app = ctx.obj.app
flower_app = Flower(capp=app, options=options, **settings)
atexit.register(flower_app.stop)
signal.signal(signal.SIGTERM, sigterm_handler)
if not ctx.obj.quiet:
print_banner(app, 'ssl_options' in settings)
try:
flower_app.start()
except (KeyboardInterrupt, SystemExit):
pass
def apply_env_options():
"apply options passed through environment variables"
env_options = filter(is_flower_envvar, os.environ)
for env_var_name in env_options:
name = env_var_name.replace(ENV_VAR_PREFIX, '', 1).lower()
value = os.environ[env_var_name]
try:
option = options._options[name] # pylint: disable=protected-access
except KeyError:
option = options._options[name.replace('_', '-')] # pylint: disable=protected-access
if option.multiple:
value = [option.type(i) for i in value.split(',')]
else:
if option.type is bool:
value = bool(strtobool(value))
else:
value = option.type(value)
setattr(options, name, value)
def apply_options(prog_name, argv):
"apply options passed through the configuration file"
argv = list(filter(is_flower_option, argv))
# parse the command line to get --conf option
parse_command_line([prog_name] + argv)
try:
parse_config_file(os.path.abspath(options.conf), final=False)
parse_command_line([prog_name] + argv)
except IOError:
if os.path.basename(options.conf) != DEFAULT_CONFIG_FILE:
raise
def warn_about_celery_args_used_in_flower_command(ctx, flower_args):
celery_options = [option for param in ctx.parent.command.params for option in param.opts]
incorrectly_used_args = []
for arg in flower_args:
arg_name, _, _ = arg.partition("=")
if arg_name in celery_options:
incorrectly_used_args.append(arg_name)
if incorrectly_used_args:
logger.warning(
'You have incorrectly specified the following celery arguments after flower command:'
' %s. '
'Please specify them after celery command instead following this template: '
'celery [celery args] flower [flower args].', incorrectly_used_args
)
def setup_logging():
if options.debug and options.logging == 'info':
options.logging = 'debug'
enable_pretty_logging()
else:
logging.getLogger("tornado.access").addHandler(NullHandler())
logging.getLogger("tornado.access").propagate = False
def extract_settings():
settings['debug'] = options.debug
if options.cookie_secret:
settings['cookie_secret'] = options.cookie_secret
if options.url_prefix:
for name in ['login_url', 'static_url_prefix']:
settings[name] = prepend_url(settings[name], options.url_prefix)
if options.auth:
settings['oauth'] = {
'key': options.oauth2_key or os.environ.get('FLOWER_OAUTH2_KEY'),
'secret': options.oauth2_secret or os.environ.get('FLOWER_OAUTH2_SECRET'),
'redirect_uri': options.oauth2_redirect_uri or os.environ.get('FLOWER_OAUTH2_REDIRECT_URI'),
}
if options.certfile and options.keyfile:
settings['ssl_options'] = dict(certfile=abs_path(options.certfile),
keyfile=abs_path(options.keyfile))
if options.ca_certs:
settings['ssl_options']['ca_certs'] = abs_path(options.ca_certs)
if options.auth and not validate_auth_option(options.auth):
logger.error("Invalid '--auth' option: %s", options.auth)
sys.exit(1)
def is_flower_option(arg):
name, _, _ = arg.lstrip('-').partition("=")
name = name.replace('-', '_')
return hasattr(options, name)
def is_flower_envvar(name):
return name.startswith(ENV_VAR_PREFIX) and \
name[len(ENV_VAR_PREFIX):].lower() in default_options
def print_banner(app, ssl):
if not options.unix_socket:
if options.url_prefix:
prefix_str = f'/{options.url_prefix}/'
else:
prefix_str = ''
logger.info(
"Visit me at http%s://%s:%s%s", 's' if ssl else '',
options.address or '0.0.0.0', options.port,
prefix_str
)
else:
logger.info("Visit me via unix socket file: %s", options.unix_socket)
logger.info('Broker: %s', app.connection().as_uri())
logger.info(
'Registered tasks: \n%s',
pformat(sorted(app.tasks.keys()))
)
logger.debug('Settings: %s', pformat(settings))
@@ -0,0 +1,210 @@
import collections
import logging
import shelve
import threading
import time
from collections import Counter
from functools import partial
from celery.events import EventReceiver
from celery.events.state import State
from prometheus_client import Counter as PrometheusCounter
from prometheus_client import Gauge, Histogram
from tornado.ioloop import PeriodicCallback
from tornado.options import options
logger = logging.getLogger(__name__)
PROMETHEUS_METRICS = None
def get_prometheus_metrics():
global PROMETHEUS_METRICS # pylint: disable=global-statement
if PROMETHEUS_METRICS is None:
PROMETHEUS_METRICS = PrometheusMetrics()
return PROMETHEUS_METRICS
class PrometheusMetrics:
def __init__(self):
self.events = PrometheusCounter('flower_events_total', "Number of events", ['worker', 'type', 'task'])
self.runtime = Histogram(
'flower_task_runtime_seconds',
"Task runtime",
['worker', 'task'],
buckets=options.task_runtime_metric_buckets
)
self.prefetch_time = Gauge(
'flower_task_prefetch_time_seconds',
"The time the task spent waiting at the celery worker to be executed.",
['worker', 'task']
)
self.number_of_prefetched_tasks = Gauge(
'flower_worker_prefetched_tasks',
'Number of tasks of given type prefetched at a worker',
['worker', 'task']
)
self.worker_online = Gauge('flower_worker_online', "Worker online status", ['worker'])
self.worker_number_of_currently_executing_tasks = Gauge(
'flower_worker_number_of_currently_executing_tasks',
"Number of tasks currently executing at a worker",
['worker']
)
class EventsState(State):
# EventsState object is created and accessed only from ioloop thread
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.counter = collections.defaultdict(Counter)
self.metrics = get_prometheus_metrics()
def event(self, event):
# Save the event
super().event(event)
worker_name = event['hostname']
event_type = event['type']
self.counter[worker_name][event_type] += 1
if event_type.startswith('task-'):
task_id = event['uuid']
task = self.tasks.get(task_id)
task_name = event.get('name', '')
if not task_name and task_id in self.tasks:
task_name = task.name or ''
self.metrics.events.labels(worker_name, event_type, task_name).inc()
runtime = event.get('runtime', 0)
if runtime:
self.metrics.runtime.labels(worker_name, task_name).observe(runtime)
task_started = task.started
task_received = task.received
if event_type == 'task-received' and not task.eta and task_received:
self.metrics.number_of_prefetched_tasks.labels(worker_name, task_name).inc()
if event_type == 'task-started' and not task.eta and task_started and task_received:
self.metrics.prefetch_time.labels(worker_name, task_name).set(task_started - task_received)
self.metrics.number_of_prefetched_tasks.labels(worker_name, task_name).dec()
if event_type in ['task-succeeded', 'task-failed'] and not task.eta and task_started and task_received:
self.metrics.prefetch_time.labels(worker_name, task_name).set(0)
if event_type == 'worker-online':
self.metrics.worker_online.labels(worker_name).set(1)
if event_type == 'worker-heartbeat':
self.metrics.worker_online.labels(worker_name).set(1)
num_executing_tasks = event.get('active')
if num_executing_tasks is not None:
self.metrics.worker_number_of_currently_executing_tasks.labels(worker_name).set(num_executing_tasks)
if event_type == 'worker-offline':
self.metrics.worker_online.labels(worker_name).set(0)
class Events(threading.Thread):
events_enable_interval = 5000
# pylint: disable=too-many-arguments
def __init__(self, capp, io_loop, db=None, persistent=False,
enable_events=True, state_save_interval=0,
**kwargs):
threading.Thread.__init__(self)
self.daemon = True
self.io_loop = io_loop
self.capp = capp
self.db = db
self.persistent = persistent
self.enable_events = enable_events
self.state = None
self.state_save_timer = None
if self.persistent:
logger.debug("Loading state from '%s'...", self.db)
state = shelve.open(self.db)
if state:
self.state = state['events']
state.close()
if state_save_interval:
self.state_save_timer = PeriodicCallback(self.save_state,
state_save_interval)
if not self.state:
self.state = EventsState(**kwargs)
self.timer = PeriodicCallback(self.on_enable_events,
self.events_enable_interval)
def start(self):
threading.Thread.start(self)
if self.enable_events:
logger.debug("Starting enable events timer...")
self.timer.start()
if self.state_save_timer:
logger.debug("Starting state save timer...")
self.state_save_timer.start()
def stop(self):
if self.enable_events:
logger.debug("Stopping enable events timer...")
self.timer.stop()
if self.state_save_timer:
logger.debug("Stopping state save timer...")
self.state_save_timer.stop()
if self.persistent:
self.save_state()
def run(self):
try_interval = 1
while True:
try:
try_interval *= 2
with self.capp.connection() as conn:
recv = EventReceiver(conn,
handlers={"*": self.on_event},
app=self.capp)
try_interval = 1
logger.debug("Capturing events...")
recv.capture(limit=None, timeout=None, wakeup=True)
except (KeyboardInterrupt, SystemExit):
try:
import _thread as thread
except ImportError:
import thread
thread.interrupt_main()
except Exception as e:
logger.error("Failed to capture events: '%s', "
"trying again in %s seconds.",
e, try_interval)
logger.debug(e, exc_info=True)
time.sleep(try_interval)
def save_state(self):
logger.debug("Saving state to '%s'...", self.db)
state = shelve.open(self.db, flag='n')
state['events'] = self.state
state.close()
def on_enable_events(self):
# Periodically enable events for workers
# launched after flower
self.io_loop.run_in_executor(None, self.capp.control.enable_events)
def on_event(self, event):
# Call EventsState.event in ioloop thread to avoid synchronization
self.io_loop.add_callback(partial(self.state.event, event))
@@ -0,0 +1,48 @@
import collections
import logging
import time
from functools import partial
logger = logging.getLogger(__name__)
class Inspector:
methods = ('stats', 'active_queues', 'registered', 'scheduled',
'active', 'reserved', 'revoked', 'conf')
def __init__(self, io_loop, capp, timeout):
self.io_loop = io_loop
self.capp = capp
self.timeout = timeout
self.workers = collections.defaultdict(dict)
def inspect(self, workername=None):
feutures = []
for method in self.methods:
feutures.append(self.io_loop.run_in_executor(None, partial(self._inspect, method, workername)))
return feutures
def _on_update(self, workername, method, response):
info = self.workers[workername]
info[method] = response
info['timestamp'] = time.time()
def _inspect(self, method, workername):
destination = [workername] if workername else None
inspect = self.capp.control.inspect(timeout=self.timeout, destination=destination)
logger.debug('Sending %s inspect command', method)
start = time.time()
result = (
getattr(inspect, method)()
if method != 'active'
else getattr(inspect, method)(safe=True)
)
logger.debug("Inspect command %s took %.2fs to complete", method, time.time() - start)
if result is None or 'error' in result:
logger.warning("Inspect method %s failed", method)
return
for worker, response in result.items():
if response is not None:
self.io_loop.add_callback(partial(self._on_update, worker, method, response))
@@ -0,0 +1,73 @@
import types
from secrets import token_urlsafe
from prometheus_client import Histogram
from tornado.options import define, options
DEFAULT_CONFIG_FILE = 'flowerconfig.py'
define("port", default=5555,
help="run on the given port", type=int)
define("address", default='',
help="run on the given address", type=str)
define("unix_socket", default='',
help="path to unix socket to bind", type=str)
define("debug", default=False,
help="run in debug mode", type=bool)
define("inspect_timeout", default=1000.0, type=float,
help="inspect timeout (in milliseconds)")
define("auth", default='', type=str,
help="regexp of emails to grant access")
define("basic_auth", type=str, default=None, multiple=True,
help="enable http basic authentication")
define("oauth2_key", type=str, default=None,
help="OAuth2 key (requires --auth)")
define("oauth2_secret", type=str, default=None,
help="OAuth2 secret (requires --auth)")
define("oauth2_redirect_uri", type=str, default=None,
help="OAuth2 redirect uri (requires --auth)")
define("max_workers", type=int, default=5000,
help="maximum number of workers to keep in memory")
define("max_tasks", type=int, default=100000,
help="maximum number of tasks to keep in memory")
define("db", type=str, default='flower',
help="flower database file")
define("persistent", type=bool, default=False,
help="enable persistent mode")
define("state_save_interval", type=int, default=0,
help="state save interval (in milliseconds)")
define("broker_api", type=str, default=None,
help="inspect broker e.g. http://guest:guest@localhost:15672/api/")
define("ca_certs", type=str, default=None,
help="SSL certificate authority (CA) file")
define("certfile", type=str, default=None,
help="SSL certificate file")
define("keyfile", type=str, default=None,
help="SSL key file")
define("xheaders", type=bool, default=False,
help="enable support for the 'X-Real-Ip' and 'X-Scheme' headers.")
define("auto_refresh", default=True,
help="refresh workerss", type=bool)
define("purge_offline_workers", default=None, type=int,
help="time (in seconds) after which offline workers are purged from workers")
define("cookie_secret", type=str, default=token_urlsafe(64),
help="secure cookie secret")
define("conf", default=DEFAULT_CONFIG_FILE,
help="configuration file")
define("enable_events", type=bool, default=True,
help="periodically enable Celery events")
define("format_task", type=types.FunctionType, default=None,
help="use custom task formatter")
define("natural_time", type=bool, default=False,
help="show time in relative format")
define("tasks_columns", type=str,
default="name,uuid,state,args,kwargs,result,received,started,runtime,worker",
help="slugs of columns on /tasks/ page, delimited by comma")
define("auth_provider", default=None, type=str, help="auth handler class")
define("url_prefix", type=str, help="base url prefix")
define("task_runtime_metric_buckets", type=float, default=Histogram.DEFAULT_BUCKETS,
multiple=True, help="histogram latency bucket value")
default_options = options
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,41 @@
.bg-green {
background-color: #f0ffeb;
}
.dataTables_wrapper {
border: 1px solid #c7ecb8;
}
.dataTables_filter input {
width: 50%;
text-indent: 5px;
}
.dataTables_length {
margin: 10px;
}
div.dataTables_wrapper .dataTables_filter input {
width: 100%;
margin: 10px;
border: 1px solid #c7ecb8;
}
.dataTables_info {
padding: 5px;
}
@media (min-width: 768px) {
div.dataTables_wrapper .dataTables_filter input {
width: 500px;
}
}
.overflow-auto {
max-width: 400px;
text-overflow: ellipsis;
}
.overflow-auto::-webkit-scrollbar {
display: none;
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,692 @@
/*jslint browser: true */
/*global $, WebSocket, jQuery */
var flower = (function () {
"use strict";
var alertContainer = document.getElementById('alert-container');
function show_alert(message, type) {
var wrapper = document.createElement('div');
wrapper.innerHTML = `
<div class="alert alert-${type} alert-dismissible" role="alert">
<div>${message}</div>
<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
</div>`;
alertContainer.appendChild(wrapper);
}
function url_prefix() {
var prefix = $('#url_prefix').val();
if (prefix) {
prefix = prefix.replace(/\/+$/, '');
if (prefix.startsWith('/')) {
return prefix;
} else {
return '/' + prefix;
}
}
return '';
}
//https://github.com/DataTables/DataTables/blob/1.10.11/media/js/jquery.dataTables.js#L14882
function htmlEscapeEntities(d) {
return typeof d === 'string' ?
d.replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;') :
d;
}
function active_page(name) {
var pathname = $(location).attr('pathname');
if (name === '/') {
return pathname === (url_prefix() + name);
}
else {
return pathname.startsWith(url_prefix() + name);
}
}
$('#worker-refresh').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
$('.dropdown-toggle').dropdown('hide');
var workername = $('#workername').text();
$.ajax({
type: 'GET',
url: url_prefix() + '/api/workers',
dataType: 'json',
data: {
workername: unescape(workername),
refresh: 1
},
success: function (data) {
show_alert(data.message || 'Successfully refreshed', 'success');
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-refresh-all').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
$('.dropdown-toggle').dropdown('hide');
$.ajax({
type: 'GET',
url: url_prefix() + '/api/workers',
dataType: 'json',
data: {
refresh: 1
},
success: function (data) {
show_alert(data.message || 'Refreshed All Workers', 'success');
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-pool-restart').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
$('.dropdown-toggle').dropdown('hide');
var workername = $('#workername').text();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/worker/pool/restart/' + workername,
dataType: 'json',
data: {
workername: workername
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-shutdown').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
$('.dropdown-toggle').dropdown('hide');
var workername = $('#workername').text();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/worker/shutdown/' + workername,
dataType: 'json',
data: {
workername: workername
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-pool-grow').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
var workername = $('#workername').text(),
grow_size = $('#pool-size option:selected').html();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/worker/pool/grow/' + workername,
dataType: 'json',
data: {
'workername': workername,
'n': grow_size,
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-pool-shrink').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
var workername = $('#workername').text(),
shrink_size = $('#pool-size option:selected').html();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/worker/pool/shrink/' + workername,
dataType: 'json',
data: {
'workername': workername,
'n': shrink_size,
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-pool-autoscale').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
var workername = $('#workername').text(),
min = $('#min-autoscale').val(),
max = $('#max-autoscale').val();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/worker/pool/autoscale/' + workername,
dataType: 'json',
data: {
'workername': workername,
'min': min,
'max': max,
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-add-consumer').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
var workername = $('#workername').text(),
queue = $('#add-consumer-name').val();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/worker/queue/add-consumer/' + workername,
dataType: 'json',
data: {
'workername': workername,
'queue': queue,
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#worker-queues').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
if (!event.target.id.startsWith("worker-cancel-consumer")) {
return;
}
var workername = $('#workername').text(),
queue = $(event.target).closest("tr").children("td:eq(0)").text();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/worker/queue/cancel-consumer/' + workername,
dataType: 'json',
data: {
'workername': workername,
'queue': queue,
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#limits-table').on('click', function (event) {
if (event.target.id.startsWith("task-timeout-")) {
var timeout = parseInt($(event.target).siblings().closest("input").val()),
type = $(event.target).text().toLowerCase(),
taskname = $(event.target).closest("tr").children("td:eq(0)").text(),
post_data = {'workername': $('#workername').text()};
taskname = taskname.split(' ')[0]; // removes [rate_limit=xxx]
post_data[type] = timeout;
if (!Number.isInteger(timeout)) {
show_alert("Invalid timeout value", "danger");
return;
}
$.ajax({
type: 'POST',
url: url_prefix() + '/api/task/timeout/' + taskname,
dataType: 'json',
data: post_data,
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert($(data.responseText).text(), "danger");
}
});
} else if (event.target.id.startsWith("task-rate-limit-")) {
var taskname = $(event.target).closest("tr").children("td:eq(0)").text(),
workername = $('#workername').text(),
ratelimit = parseInt($(event.target).prev().val());
taskname = taskname.split(' ')[0]; // removes [rate_limit=xxx]
$.ajax({
type: 'POST',
url: url_prefix() + '/api/task/rate-limit/' + taskname,
dataType: 'json',
data: {
'workername': workername,
'ratelimit': ratelimit,
},
success: function (data) {
show_alert(data.message, "success");
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
}
});
$('#task-revoke').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
var taskid = $('#taskid').text();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/task/revoke/' + taskid,
dataType: 'json',
data: {
'terminate': false,
},
success: function (data) {
show_alert(data.message, "success");
document.getElementById("task-revoke").disabled = true;
setTimeout(function() {location.reload();}, 5000);
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
$('#task-terminate').on('click', function (event) {
event.preventDefault();
event.stopPropagation();
var taskid = $('#taskid').text();
$.ajax({
type: 'POST',
url: url_prefix() + '/api/task/revoke/' + taskid,
dataType: 'json',
data: {
'terminate': true,
},
success: function (data) {
show_alert(data.message, "success");
document.getElementById("task-terminate").disabled = true;
setTimeout(function() {location.reload();}, 5000);
},
error: function (data) {
show_alert(data.responseText, "danger");
}
});
});
function sum(a, b) {
return parseInt(a, 10) + parseInt(b, 10);
}
function format_time(timestamp) {
var time = $('#time').val(),
prefix = time.startsWith('natural-time') ? 'natural-time' : 'time',
tz = time.substr(prefix.length + 1) || 'UTC';
if (prefix === 'natural-time') {
return moment.unix(timestamp).tz(tz).fromNow();
}
return moment.unix(timestamp).tz(tz).format('YYYY-MM-DD HH:mm:ss.SSS');
}
function isColumnVisible(name) {
var columns = $('#columns').val();
if (columns === "all")
return true;
if (columns) {
columns = columns.split(',').map(function (e) {
return e.trim();
});
return columns.indexOf(name) !== -1;
}
return true;
}
$.urlParam = function (name) {
var results = new RegExp('[\\?&]' + name + '=([^&#]*)').exec(window.location.href);
return (results && results[1]) || 0;
};
$(document).ready(function () {
//https://github.com/twitter/bootstrap/issues/1768
var shiftWindow = function () {
scrollBy(0, -50);
};
if (location.hash) {
shiftWindow();
}
window.addEventListener("hashchange", shiftWindow);
// Make bootstrap tabs persistent
$(document).ready(function () {
if (location.hash !== '') {
$('a[href="' + location.hash + '"]').tab('show');
}
// Listen for tab shown events and update the URL hash fragment accordingly
$('.nav-tabs a[data-bs-toggle="tab"]').on('shown.bs.tab', function (event) {
const tabPaneId = $(event.target).attr('href').substr(1);
if (tabPaneId) {
window.location.hash = tabPaneId;
}
});
});
});
$(document).ready(function () {
if (!active_page('/') && !active_page('/workers')) {
return;
}
$('#workers-table').DataTable({
rowId: 'name',
searching: true,
select: false,
paging: true,
scrollCollapse: true,
lengthMenu: [15, 30, 50, 100],
pageLength: 15,
language: {
lengthMenu: 'Show _MENU_ workers',
info: 'Showing _START_ to _END_ of _TOTAL_ workers',
infoFiltered: '(filtered from _MAX_ total workers)'
},
ajax: url_prefix() + '/workers?json=1',
order: [
[1, "des"]
],
footerCallback: function( tfoot, data, start, end, display ) {
var api = this.api();
var columns = {2:"STARTED", 3:"", 4:"FAILURE", 5:"SUCCESS", 6:"RETRY"};
for (const [column, state] of Object.entries(columns)) {
var total = api.column(column).data().reduce(sum, 0);
var footer = total;
if (total !== 0) {
let queryParams = (state !== '' ? `?state=${state}` : '');
footer = '<a href="' + url_prefix() + '/tasks' + queryParams + '">' + total + '</a>';
}
$(api.column(column).footer()).html(footer);
}
},
columnDefs: [{
targets: 0,
data: 'hostname',
type: 'natural',
render: function (data, type, full, meta) {
return '<a href="' + url_prefix() + '/worker/' + encodeURIComponent(data) + '">' + data + '</a>';
}
}, {
targets: 1,
data: 'status',
className: "text-center",
width: "10%",
render: function (data, type, full, meta) {
if (data) {
return '<span class="badge bg-success">Online</span>';
} else {
return '<span class="badge bg-secondary">Offline</span>';
}
}
}, {
targets: 2,
data: 'active',
className: "text-center",
width: "10%",
defaultContent: 0
}, {
targets: 3,
data: 'task-received',
className: "text-center",
width: "10%",
defaultContent: 0
}, {
targets: 4,
data: 'task-failed',
className: "text-center",
width: "10%",
defaultContent: 0
}, {
targets: 5,
data: 'task-succeeded',
className: "text-center",
width: "10%",
defaultContent: 0
}, {
targets: 6,
data: 'task-retried',
className: "text-center",
width: "10%",
defaultContent: 0
}, {
targets: 7,
data: 'loadavg',
width: "10%",
className: "text-center text-nowrap",
render: function (data, type, full, meta) {
if (!full.status) {
return 'N/A';
}
if (Array.isArray(data)) {
return data.join(', ');
}
return data;
}
}, ],
});
var autorefresh_interval = $.urlParam('autorefresh') || 1;
if (autorefresh !== 0) {
setInterval( function () {
$('#workers-table').DataTable().ajax.reload(null, false);
}, autorefresh_interval * 1000);
}
});
$(document).ready(function () {
if (!active_page('/tasks')) {
return;
}
$('#tasks-table').DataTable({
rowId: 'uuid',
searching: true,
scrollX: true,
scrollCollapse: true,
processing: true,
serverSide: true,
colReorder: true,
lengthMenu: [15, 30, 50, 100],
pageLength: 15,
language: {
lengthMenu: 'Show _MENU_ tasks',
info: 'Showing _START_ to _END_ of _TOTAL_ tasks',
infoFiltered: '(filtered from _MAX_ total tasks)'
},
ajax: {
type: 'POST',
url: url_prefix() + '/tasks/datatable'
},
order: [
[7, "desc"]
],
oSearch: {
"sSearch": $.urlParam('state') ? 'state:' + $.urlParam('state') : ''
},
columnDefs: [{
targets: 0,
data: 'name',
visible: isColumnVisible('name'),
render: function (data, type, full, meta) {
return data;
}
}, {
targets: 1,
data: 'uuid',
visible: isColumnVisible('uuid'),
orderable: false,
className: "text-nowrap",
render: function (data, type, full, meta) {
return '<a href="' + url_prefix() + '/task/' + encodeURIComponent(data) + '">' + data + '</a>';
}
}, {
targets: 2,
data: 'state',
visible: isColumnVisible('state'),
className: "text-center",
render: function (data, type, full, meta) {
switch (data) {
case 'SUCCESS':
return '<span class="badge bg-success">' + data + '</span>';
case 'FAILURE':
return '<span class="badge bg-danger">' + data + '</span>';
default:
return '<span class="badge bg-secondary">' + data + '</span>';
}
}
}, {
targets: 3,
data: 'args',
className: "text-nowrap overflow-auto",
visible: isColumnVisible('args'),
render: htmlEscapeEntities
}, {
targets: 4,
data: 'kwargs',
className: "text-nowrap overflow-auto",
visible: isColumnVisible('kwargs'),
render: htmlEscapeEntities
}, {
targets: 5,
data: 'result',
visible: isColumnVisible('result'),
className: "text-nowrap overflow-auto",
render: htmlEscapeEntities
}, {
targets: 6,
data: 'received',
className: "text-nowrap",
visible: isColumnVisible('received'),
render: function (data, type, full, meta) {
if (data) {
return format_time(data);
}
return data;
}
}, {
targets: 7,
data: 'started',
className: "text-nowrap",
visible: isColumnVisible('started'),
render: function (data, type, full, meta) {
if (data) {
return format_time(data);
}
return data;
}
}, {
targets: 8,
data: 'runtime',
className: "text-center",
visible: isColumnVisible('runtime'),
render: function (data, type, full, meta) {
return data ? data.toFixed(2) : data;
}
}, {
targets: 9,
data: 'worker',
visible: isColumnVisible('worker'),
render: function (data, type, full, meta) {
return '<a href="' + url_prefix() + '/worker/' + encodeURIComponent(data) + '">' + data + '</a>';
}
}, {
targets: 10,
data: 'exchange',
visible: isColumnVisible('exchange')
}, {
targets: 11,
data: 'routing_key',
visible: isColumnVisible('routing_key')
}, {
targets: 12,
data: 'retries',
className: "text-center",
visible: isColumnVisible('retries')
}, {
targets: 13,
data: 'revoked',
className: "text-nowrap",
visible: isColumnVisible('revoked'),
render: function (data, type, full, meta) {
if (data) {
return format_time(data);
}
return data;
}
}, {
targets: 14,
data: 'exception',
className: "text-nowrap",
visible: isColumnVisible('exception')
}, {
targets: 15,
data: 'expires',
visible: isColumnVisible('expires')
}, {
targets: 16,
data: 'eta',
visible: isColumnVisible('eta')
}, ],
});
});
}(jQuery));
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,523 @@
{
"tags": [
],
"paths": {
"\/api\/tasks": {
"get": {
"responses": {
"200": {
"description": "Result"
}
},
"description": "List tasks",
"parameters": [
{
"name": "limit",
"in": "query",
"description": "the maximum number of tasks",
"required": false,
"format": "int32",
"type": "integer"
},
{
"name": "workername",
"in": "query",
"description": "filter task by workername",
"required": false,
"type": "string"
},
{
"name": "taskname",
"in": "query",
"description": "filter task by taskname",
"required": false,
"type": "string"
},
{
"name": "state",
"in": "query",
"description": "filter task by state",
"required": false,
"type": "string"
}
]
}
},
"\/api\/task\/types": {
"get": {
"responses": {
"200": {
"description": "result"
}
},
"description": "List (seen) task types"
}
},
"\/api\/queues\/length": {
"get": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Get queue lengths"
}
},
"\/api\/task\/info\/{taskid}": {
"get": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Get task info",
"parameters": [
{
"$ref": "#\/parameters\/taskid"
}
]
}
},
"\/api\/task\/apply\/{taskname}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Execute a task by name and wait results",
"parameters": [
{
"$ref": "#\/parameters\/taskname"
},
{
"schema": {
"type": "object",
"properties": {
"kwargs": {
"type": "object"
},
"args": {
"type": "array"
}
}
},
"name": "args",
"description": "the dictionary of args and kwargs",
"in": "body"
}
]
}
},
"\/api\/task\/async-apply\/{taskname}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Execute a task",
"parameters": [
{
"$ref": "#\/parameters\/taskname"
},
{
"schema": {
"type": "object",
"properties": {
"kwargs": {
"type": "object"
},
"args": {
"type": "array"
},
"options": {
"type": "object"
}
}
},
"name": "args",
"description": "the dictionary of args, kwargs, and apply-async options",
"in": "body"
}
]
}
},
"\/api\/task\/send-task\/{taskname}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Execute a task by name (Doesn't require a task source)",
"parameters": [
{
"$ref": "#\/parameters\/taskname"
},
{
"schema": {
"type": "object",
"properties": {
"kwargs": {
"type": "object"
},
"args": {
"type": "array"
}
}
},
"name": "args",
"description": "the dictionary of args, and kwargs",
"in": "body"
}
]
}
},
"\/api\/task\/result\/{taskid}": {
"get": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Get a task result",
"parameters": [
{
"$ref": "#\/parameters\/taskid"
},
{
"name": "timeout",
"in": "query",
"description": "how long to wait, in seconds, before the operation times out",
"required": false,
"format": "int32",
"type": "integer"
}
]
}
},
"\/api\/task\/abort\/{taskid}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Abort a running task",
"parameters": [
{
"$ref": "#\/parameters\/taskid"
}
]
}
},
"\/api\/task\/timeout\/{taskname}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Change soft and hard time limits for a task",
"parameters": [
{
"$ref": "#\/parameters\/taskname"
},
{
"name": "workername",
"in": "query",
"description": "the name of a worker",
"required": true,
"type": "string"
},
{
"name": "soft",
"in": "query",
"description": "the soft timeout limit",
"required": false,
"format": "int32",
"type": "integer"
},
{
"name": "hard",
"in": "query",
"description": "the hard timeout limit",
"required": false,
"format": "int32",
"type": "integer"
}
]
}
},
"\/api\/task\/rate-limit\/{taskname}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Change rate limit for a task",
"parameters": [
{
"$ref": "#\/parameters\/taskname"
},
{
"name": "workername",
"in": "query",
"description": "the name of a worker",
"required": true,
"type": "string"
},
{
"name": "rateLimit",
"in": "query",
"description": "the rate limit to apply",
"required": true,
"format": "int32",
"type": "integer"
}
]
}
},
"\/api\/task\/revoke\/{taskid}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Revoke a task",
"parameters": [
{
"$ref": "#\/parameters\/taskid"
},
{
"name": "terminate",
"in": "query",
"description": "terminate the task if it is running",
"required": false,
"type": "boolean"
}
]
}
},
"\/api\/workers": {
"get": {
"responses": {
"200": {
"description": "result"
}
},
"description": "List workers",
"parameters": [
{
"name": "refresh",
"in": "query",
"description": "run inspect to get updated list of workers",
"required": false,
"type": "boolean"
},
{
"name": "workername",
"in": "query",
"description": "get info for workername",
"required": false,
"type": "string"
},
{
"name": "status",
"description": "only get worker status info",
"in": "query",
"type": "boolean"
}
]
}
},
"\/api\/worker\/shutdown\/{workername}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Shut down a worker",
"parameters": [
{
"$ref": "#\/parameters\/workername"
}
]
}
},
"\/api\/worker\/pool\/restart\/{workername}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Restart a worker's pool",
"parameters": [
{
"$ref": "#\/parameters\/workername"
}
]
}
},
"\/api\/worker\/pool\/grow\/{workername}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Grow a worker's pool",
"parameters": [
{
"$ref": "#\/parameters\/workername"
},
{
"name": "n",
"in": "query",
"description": "number of pool processes to grow, default is 1",
"required": false,
"format": "int32",
"type": "integer"
}
]
}
},
"\/api\/worker\/pool\/shrink\/{workername}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Shrink a worker's pool",
"parameters": [
{
"$ref": "#\/parameters\/workername"
},
{
"name": "n",
"in": "query",
"description": "number of pool processes to shrink, default is 1",
"required": false,
"format": "int32",
"type": "integer"
}
]
}
},
"\/api\/worker\/pool\/autoscale\/{workername}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Autoscale a worker pool",
"parameters": [
{
"$ref": "#\/parameters\/workername"
},
{
"name": "min",
"in": "query",
"description": "minimum number of pool processes",
"required": false,
"format": "int32",
"type": "integer"
},
{
"name": "max",
"in": "query",
"description": "maximum number of pool processes",
"required": false,
"format": "int32",
"type": "integer"
}
]
}
},
"\/api\/worker\/queue\/add-consumer\/{workername}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Start consuming from a queue",
"parameters": [
{
"$ref": "#\/parameters\/workername"
},
{
"name": "queue",
"in": "query",
"description": "the name of a queue",
"required": true,
"type": "string"
}
]
}
},
"\/api\/worker\/queue\/cancel-consumer\/{workername}": {
"post": {
"responses": {
"200": {
"description": "result"
}
},
"description": "Stop consuming from a queue",
"parameters": [
{
"$ref": "#\/parameters\/workername"
},
{
"name": "queue",
"in": "query",
"description": "the name of a queue",
"required": true,
"type": "string"
}
]
}
}
},
"parameters": {
"taskid": {
"type": "string",
"in": "path",
"description": "The task id",
"required": true,
"name": "taskid"
},
"workername": {
"type": "string",
"in": "path",
"description": "The worker name",
"required": true,
"name": "workername"
},
"taskname": {
"type": "string",
"in": "path",
"description": "The task name",
"required": true,
"name": "taskname"
}
},
"info": {
"description": "The flower API spec",
"version": "1.0.0-dev",
"title": "Flower"
},
"definitions": {
},
"swagger": "2.0"
}
@@ -0,0 +1,13 @@
{% extends "base.html" %}
{% block container %}
<div class="col-12">
<p>
{% if message %}
{{ message }}
{% else %}
Error, page not found
{% end %}
</p>
</div>
{% end %}
@@ -0,0 +1,35 @@
{% import pprint %}
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Flower</title>
<link rel="stylesheet" href="{{ static_url('css/bootstrap.min.css') }}">
<link rel="stylesheet" href="{{ static_url('css/datatables-1.13.4.min.css') }}">
<link href="{{ static_url('css/flower.css') }}" rel="stylesheet">
</head>
<body class="m-2">
{% block navbar %}
{% module Template("navbar.html", active_tab="") %}
{% end %}
<div class="container-fluid my-2">
<div id="alert-container"></div>
<input type="hidden" value="{{ url_prefix or '' }}" id='url_prefix'>
</div>
{% block container %}
{% end %}
<script src="{{ static_url('js/bootstrap.bundle.min.js') }}"></script>
<script src="{{ static_url('js/jquery-3.6.4.min.js') }}"></script>
<script src="{{ static_url('js/datatables-1.13.4.min.js') }}"></script>
<script src="{{ static_url('js/moment-2.29.4.min.js') }}"></script>
<script src="{{ static_url('js/moment-timezone-with-data-2.29.4.min.js') }}"></script>
<script src="{{ static_url('js/flower.js') }}"></script>
{% block extra_scripts %}
{% end %}
</body>
</html>
@@ -0,0 +1,37 @@
{% extends "base.html" %}
{% block navbar %}
{% module Template("navbar.html", active_tab="broker") %}
{% end %}
{% block container %}
<div class="container-fluid">
<figure class="table-responsive mt-3">
<table id="queue-table" class="table table-bordered table-striped caption-top">
<caption>{{ broker_url }}</caption>
<thead>
<tr>
<th>Queue</th>
<th>Messages</th>
<th>Unacked</th>
<th>Ready</th>
<th>Consumers</th>
<th>Idle since</th>
</tr>
</thead>
<tbody>
{% for queue in queues %}
<tr id="{{ url_escape(queue['name']) }}">
<td>{{ queue['name'] }}</td>
<td>{{ queue.get('messages', 'N/A') }}</td>
<td>{{ queue.get('messages_unacknowledged', 'N/A') }}</td>
<td>{{ queue.get('messages_ready', 'N/A') }}</td>
<td>{{ queue.get('consumers', 'N/A') }}</td>
<td>{{ queue.get('idle_since', 'N/A') }}</td>
</tr>
{% end %}
</tbody>
</table>
</figure>
</div>
{% end %}
@@ -0,0 +1,20 @@
{% extends "base.html" %}
{% block container %}
{% if debug %}
<div class="col-12">
<p>It looks like you have found a bug! You can help to improve
Flower by opening an issue in <a href="https://github.com/mher/flower/issues">https://github.com/mher/flower/issues</a>
</p>
<pre>
{{ bugreport }}
{{ error_trace }}
</pre>
</div>
{% else %}
<div class="col-12">
Error {{ status_code }}
</div>
{% end %}
{% end %}
@@ -0,0 +1,41 @@
<nav class="navbar navbar-expand-lg navbar-light bg-green mx-2">
<a class="navbar-brand" href="{{ reverse_url('main') }}">
<img src="{{ static_url('favicon.ico') }}" width="30" height="30" class="d-inline-block align-top" alt="">
Flower
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent"
aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav mr-auto">
<li class="nav-item active">
<a class="nav-link text-dark" href="{{ reverse_url('workers') }}">Workers</a>
</li>
<li class="nav-item">
<a class="nav-link text-dark" href="{{ reverse_url('tasks') }}">Tasks</a>
</li>
<li class="nav-item">
<a class="nav-link text-dark" href="{{ reverse_url('broker') }}">Broker</a>
</li>
<li class="nav-item">
<a class="nav-link text-dark" href="https://flower.readthedocs.io/" target="_blank" rel="noopener">Documentation</a>
</li>
</ul>
<ul class="navbar-nav flex-row flex-wrap ms-md-auto">
<li class="nav-item col-6 col-lg-auto">
<a class="nav-link py-2 px-0 px-lg-2" href="https://github.com/mher/flower" target="_blank" rel="noopener">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="30" class="navbar-nav-svg" viewBox="0 0 512 499.36"
role="img">
<path fill="currentColor" fill-rule="evenodd"
d="M256 0C114.64 0 0 114.61 0 256c0 113.09 73.34 209 175.08 242.9 12.8 2.35 17.47-5.56 17.47-12.34 0-6.08-.22-22.18-.35-43.54-71.2 15.49-86.2-34.34-86.2-34.34-11.64-29.57-28.42-37.45-28.42-37.45-23.27-15.84 1.73-15.55 1.73-15.55 25.69 1.81 39.21 26.38 39.21 26.38 22.84 39.12 59.92 27.82 74.5 21.27 2.33-16.54 8.94-27.82 16.25-34.22-56.84-6.43-116.6-28.43-116.6-126.49 0-27.95 10-50.8 26.35-68.69-2.63-6.48-11.42-32.5 2.51-67.75 0 0 21.49-6.88 70.4 26.24a242.65 242.65 0 0 1 128.18 0c48.87-33.13 70.33-26.24 70.33-26.24 14 35.25 5.18 61.27 2.55 67.75 16.41 17.9 26.31 40.75 26.31 68.69 0 98.35-59.85 120-116.88 126.32 9.19 7.9 17.38 23.53 17.38 47.41 0 34.22-.31 61.83-.31 70.23 0 6.85 4.61 14.81 17.6 12.31C438.72 464.97 512 369.08 512 256.02 512 114.62 397.37 0 256 0z" />
</svg>
<small class="d-lg-none ms-2">GitHub</small>
</a>
</li>
</ul>
</div>
</nav>
@@ -0,0 +1,92 @@
{% extends "base.html" %}
{% block navbar %}
{% module Template("navbar.html", active_tab="tasks") %}
{% end %}
{% block container %}
<div id='task-page' class="container-fluid mt-3">
<div class="row-fluid">
<div class="col-lg-12">
<div class="page-header">
<p id="taskid" class="d-none">{{ task.uuid }}</p>
<h2>{{ getattr(task, 'name', None) }}
<small class="text-muted fs-5">{{ task.uuid }}</small>
{% if task.state == "STARTED" %}
<button class="btn btn-danger float-end" id="task-terminate">Terminate</button>
{% elif task.state == "RECEIVED" or task.state == "RETRY" %}
<button class="btn btn-danger float-end" id="task-revoke">Revoke</button>
{% end %}
</h2>
</div>
<div class="row-fluid">
<div class="col-lg-6">
<table class="table table-bordered table-striped">
<tbody>
<tr>
<td>Name</td>
<td>{{ getattr(task, 'name', None) }}</td>
</tr>
<tr>
<td>UUID</td>
<td>{{ task.uuid }}</td>
</tr>
<tr>
<td>State</td>
<td>
{% if task.state == "SUCCESS" %}
<span class="badge bg-success">{{ task.state }}</span>
{% elif task.state == "FAILURE" %}
<span class="badge bg-danger">{{ task.state }}</span>
{% else %}
<span class="badge bg-secondary">{{ task.state }}</span>
{% end %}
</td>
</tr>
<tr>
<td>args</td>
<td>{{ task.args }}</td>
</tr>
<tr>
<td>kwargs</td>
<td>{{ task.kwargs }}</td>
</tr>
<tr>
<td>Result</td>
<td>{{ getattr(task, 'result', '') }}</td>
</tr>
{% for name in task._fields %}
{% if name not in ['name', 'uuid', 'state', 'args', 'kwargs', 'result'] and getattr(task, name, None) is not None %}
<tr>
<td>{{ humanize(name) }}</td>
<td>
{% if name in ['sent', 'received', 'started', 'succeeded', 'retried', 'timestamp', 'failed', 'revoked'] %}
{{ humanize(getattr(task, name, None), type='time') }}
{% elif name == 'worker' %}
<a
href="{{ reverse_url('worker', task.worker.hostname) }}">{{ task.worker.hostname }}</a>
{% elif name == 'traceback' %}
<pre>{{ getattr(task, name, None) }}</pre>
{% elif name in ['parent_id', 'root_id'] %}
<a
href="{{ reverse_url('task', getattr(task, name, None)) }}">{{ getattr(task, name, None) }}</a>
{% elif name == 'children' %}
{% for child in getattr(task, name, {}) %}
<a href="{{ reverse_url('task', child.id) }}">{{ child.id }}</a>
<br>
{% end %}
{% else %}
{{ getattr(task, name, None) }}
{% end %}
</td>
</tr>
{% end %}
{% end %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
{% end %}
@@ -0,0 +1,73 @@
{% extends "base.html" %}
{% block navbar %}
{% module Template("navbar.html", active_tab="tasks") %}
{% end %}
{% block container %}
<input type="hidden" value="{{ time }}" id='time'>
<input type="hidden" value="{{ columns }}" id='columns'>
<div class="container-fluid mt-3">
<table id="tasks-table" class="table table-bordered table-striped table-hover w-100">
<thead>
<tr>
<th>Name</th>
<th>UUID</th>
<th class="text-center">State</th>
<th>args</th>
<th>kwargs</th>
<th>Result</th>
<th class="text-center">Received</th>
<th class="text-center">Started</th>
<th class="text-center">Runtime</th>
<th>Worker</th>
<th>Exchange</th>
<th>Routing Key</th>
<th class="text-center">Retries</th>
<th class="text-center">Revoked</th>
<th>Exception</th>
<th class="text-center">Expires</th>
<th class="text-center">ETA</th>
</tr>
</thead>
<tbody>
{% for uuid, task in tasks %}
{% if getattr(task, 'name', None) is None %}
{% continue %}
{% end %}
<tr>
<td>{{ task.name }}</td>
<td>{{ task.uuid }}</td>
<td>{{ task.state }}</td>
<td>{{ task.args }}</td>
<td>{{ task.kwargs }}</td>
<td>
{% if task.state == "SUCCESS" %}
{{ task.result }}
{% elif task.state == "FAILURE" %}
{{ task.exception }}
{% end %}
</td>
<td>{{ humanize(task.received, type='time') }}</td>
<td>{{ humanize(task.started, type='time') }}</td>
<td>
{% if task.timestamp and task.started %}
{{ '%.2f' % humanize(task.timestamp - task.started) }} sec
{% end %}
</td>
<td>{{ task.worker }}</td>
<td>{{ task.exchange }}</td>
<td>{{ task.routing_key }}</td>
<td>{{ task.retries }}</td>
<td>{{ humanize(task.revoked, type='time') }}</td>
<td>{{ task.exception }}</td>
<td>{{ task.expires }}</td>
<td>{{ task.eta }}</td>
</tr>
{% end %}
</tbody>
</table>
</div>
{% end %}
@@ -0,0 +1,390 @@
{% extends "base.html" %}
{% block navbar %}
{% module Template("navbar.html", active_tab="workers") %}
{% end %}
{% block container %}
{% set other = {key: value for key, value in worker['stats'].items() if key not in
'pool pid prefetch_count autoscaler consumer broker clock total rusage'.split()} %}
<div class="container-fluid">
<div class="row-fluid">
<div class="col-lg-12">
<div class="mt-4 mb-4">
<h3 id="workername">{{ worker['name'] }}</h3>
</div>
<div class="btn-group float-end" role="group" aria-label="Button Group">
<button id="worker-group" type="button" class="btn btn-primary dropdown-toggle" data-bs-toggle="dropdown"
aria-expanded="false">
Refresh
</button>
<ul class="dropdown-menu" aria-labelledby="btnGroupDrop">
<li><a id="worker-shutdown" class="dropdown-item" data-bs-dismiss="dropdown">Shut Down</a></li>
<li><a id="worker-pool-restart" class="dropdown-item" data-bs-dismiss="dropdown">Restart Pool</a></li>
<li><a id="worker-refresh" class="dropdown-item" data-bs-dismiss="dropdown">Refresh</a></li>
<li><a id="worker-refresh-all" class="dropdown-item" data-bs-dismiss="dropdown">Refresh All</a></li>
</ul>
</div>
<div class="tabbable">
<ul class="nav nav-tabs" role="tablist">
<li class="nav-item" role="presentation"><a class="nav-link active" href="#tab-pool" data-bs-toggle="tab"
data-bs-target="#tab-pool" type="button" role="tab" aria-controls="tab-pool" aria-selected="true">Pool</a>
</li>
<li class="nav-item" role="presentation"><a class="nav-link" href="#tab-broker" data-bs-toggle="tab"
data-bs-target="#tab-broker" type="button" role="tab" aria-controls="tab-broker"
aria-selected="false">Broker</a></li>
<li class="nav-item" role="presentation"><a class="nav-link" href="#tab-queues" data-bs-toggle="tab"
data-bs-target="#tab-queues" type="button" role="tab" aria-controls="tab-queues"
aria-selected="false">Queues</a></li>
<li class="nav-item" role="presentation"><a class="nav-link" href="#tab-tasks" data-bs-toggle="tab"
data-bs-target="#tab-tasks" type="button" role="tab" aria-controls="tab-tasks"
aria-selected="false">Tasks</a></li>
<li class="nav-item" role="presentation"><a class="nav-link" href="#tab-limits" data-bs-toggle="tab"
data-bs-target="#tab-limits" type="button" role="tab" aria-controls="tab-limits"
aria-selected="false">Limits</a></li>
<li class="nav-item" role="presentation"><a class="nav-link" href="#tab-config" data-bs-toggle="tab"
data-bs-target="#tab-config" type="button" role="tab" aria-controls="tab-config"
aria-selected="false">Config</a></li>
<li class="nav-item" role="presentation"><a class="nav-link" href="#tab-system" data-bs-toggle="tab"
data-bs-target="#tab-system" type="button" role="tab" aria-controls="tab-system"
aria-selected="false">System</a></li>
{% if other %}
<li class="nav-item"><a class="nav-link" href="#tab-other" data-bs-toggle="tab" data-bs-target="#tab-other"
type="button" role="tab" aria-controls="tab-other" aria-selected="false">Other</a></li>
{% end %}
</ul>
<div class="tab-content">
<div class="tab-pane fade show active" id="tab-pool" role="tabpanel" aria-labelledby="tab-pool">
<div class="container-fluid">
<div class="row">
<div class="col-lg-8">
<table class="table table-bordered table-striped caption-top">
<caption>Worker pool options</caption>
<tbody>
{% for name,value in worker['stats'].get('pool', {}).items() %}
<tr>
<td>{{ humanize(name) }}</td>
<td>{{ humanize(value) }}</td>
</tr>
{% end %}
<tr>
<td>Worker PID</td>
<td>{{ worker['stats'].get('pid', 'N/A')}}</td>
</tr>
<tr>
<td>Prefetch Count</td>
<td>{{ worker['stats'].get('prefetch_count', 'N/A')}}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-lg-4 container">
<form class="mx-auto">
<legend class="form-label mt-md-5">Pool size control</legend>
<div class="mb-3">
<label for="pool-size" class="col-sm-2 col-form-label text-nowrap">Pool size</label>
<input type="number" id="pool-size" min="1" max="100" value="1">
<button id="worker-pool-grow" class="btn btn-primary btn-sm" type="button">Grow</button>
<button id="worker-pool-shrink" class="btn btn-primary btn-sm" type="button">Shrink</button>
</div>
<div class="mb-3 input-group">
<label for="min-autoscale" class="col-sm-2 form-label text-nowrap">Auto scale</label>
<input class="form-control-sm" id="min-autoscale" type="number" placeholder="Min" min="1"
max="100">
<input class="form-control-sm mx-1" id="max-autoscale" type="number" placeholder="Max" min="1"
max="100">
<button id="worker-pool-autoscale" class="btn btn-primary btn-sm" type="button">Apply</button>
</div>
</form>
</div>
</div>
{% if worker['stats'].get('autoscaler', None) %}
<div class="col-md-4">
<table class="table table-bordered table-striped caption-top">
<caption>Autoscaler options</caption>
<tbody>
{% for name,value in worker['stats']['autoscaler'].items() %}
<tr>
<td>{{ humanize(name) }}</td>
<td>{{ humanize(value) }}</td>
</tr>
{% end %}
</tbody>
</table>
</div>
{% end %}
</div>
</div> <!-- end pool tab -->
<div class="tab-pane fade" id="tab-broker" role="tabpanel" aria-labelledby="tab-broker">
<div class="col-lg-6">
<table class="table table-bordered table-striped caption-top">
<caption>Broker options</caption>
<tbody>
{% for name,value in (worker['stats'].get('consumer', None) or worker['stats'])['broker'].items() %}
<tr>
<td>{{ humanize(name) }}</td>
<td>{{ value }}</td>
</tr>
{% end %}
</tbody>
</table>
</div>
</div> <!-- end broker tab -->
<div class="tab-pane fade" id="tab-queues" role="tabpanel" aria-labelledby="tab-queues">
<table class="table table-bordered table-striped caption-top">
<caption>Active queues</caption>
<thead>
<tr>
<th>Name</th>
<th>Exclusive</th>
<!-- <th>Exchange</th> -->
<th>Durable</th>
<th>Routing key</th>
<th>No ACK</th>
<th>Alias</th>
<th>Queue arguments</th>
<th>Binding arguments</th>
<th>Auto delete</th>
<th style="width: 125px;"></th>
</tr>
</thead>
<tbody id="worker-queues">
{% for queue in worker.get('active_queues', []) %}
<tr>
<td>{{ queue['name'] }}</td>
<td>{{ queue['exclusive'] }}</td>
<!-- <td>{{ queue['exchange'] }}</td> -->
<td>{{ queue['durable'] }}</td>
<td>{{ queue['routing_key'] }}</td>
<td>{{ queue['no_ack'] }}</td>
<td>{{ queue['alias'] }}</td>
<td>{{ queue['queue_arguments'] }}</td>
<td>{{ queue['binding_arguments'] }}</td>
<td>{{ queue['auto_delete'] }}</td>
<td><button id="worker-cancel-consumer-{{ queue['name'] }}" class="btn btn-danger text-nowrap">Cancel
Consumer</button></td>
</tr>
{% end %}
</tbody>
</table>
<div class="control-group col-lg-3">
<div class="input-group mb-3">
<input id="add-consumer-name" type="text" class="form-control" placeholder="New consumer"
aria-label="New consumer" aria-describedby="worker-add-consumer">
<button class="btn btn-primary mx-1" type="button" id="worker-add-consumer">Add</button>
</div>
</div>
</div> <!-- end queues tab -->
<div class="tab-pane fade" id="tab-tasks" role="tabpanel" aria-labelledby="tab-tasks">
<table class="table table-bordered table-striped caption-top">
<caption>Processed tasks</caption>
<tbody>
{% for name,value in worker['stats']['total'].items() %}
<tr>
<td>{{ name }}</td>
<td>{{ value }}</td>
</tr>
{% end %}
</tbody>
</table>
<table class="table table-bordered table-striped caption-top">
<caption>Active tasks</caption>
<thead>
<tr>
<th>Name</th>
<th>UUID</th>
<th>Ack</th>
<th>PID</th>
<th>args</th>
<th>kwargs</th>
</tr>
</thead>
<tbody>
{% for task in worker.get('active', {}) %}
<tr>
<td>{{ task['name'] }}</td>
<td><a href="{{ reverse_url('task', task['id']) }}">{{ task['id'] }}</a></td>
<td>{{ task['acknowledged'] }}</td>
<td>{{ task['worker_pid'] }}</td>
<td>{{ task.get('args', 'N/A') }}</td>
<td>{{ task.get('kwargs', 'N/A') }}</td>
</tr>
{% end %}
</tbody>
</table>
<table class="table table-bordered table-striped caption-top">
<caption>Scheduled tasks</caption>
<thead>
<tr>
<th>Name</th>
<th>UUID</th>
<th>args</th>
<th>kwargs</th>
</tr>
</thead>
<tbody>
{% for task in worker.get('scheduled', {}) %}
<tr>
<td>{{ task['request']['name'] }}</td>
<td><a href="{{ reverse_url('task', task['request']['id']) }}">{{ task['request']['id'] }}</a></td>
<td>{{ task['request']['args'] }}</td>
<td>{{ task['request']['kwargs'] }}</td>
</tr>
{% end %}
</tbody>
</table>
<table class="table table-bordered table-striped caption-top">
<caption>Reserved tasks</caption>
<thead>
<tr>
<th>Name</th>
<th>UUID</th>
<th>args</th>
<th>kwargs</th>
</tr>
</thead>
<tbody>
{% for task in worker.get('reserved', {}) %}
<tr>
<td>{{ task['name'] }}</td>
<td><a href="{{ reverse_url('task', task['id']) }}">{{ task['id'] }}</a></td>
<td>{{ task['args'] }}</td>
<td>{{ task['kwargs'] }}</td>
</tr>
{% end %}
</tbody>
</table>
<table class="table table-bordered table-striped caption-top">
<caption>Revoked tasks</caption>
<thead>
<tr>
<th>UUID</th>
</tr>
</thead>
<tbody>
{% for task in worker.get('revoked', []) %}
<tr>
<td><a href="{{ reverse_url('task', task) }}">{{ task }}</a></td>
</tr>
{% end %}
</tbody>
</table>
</div> <!-- end tasks tab -->
<div class="tab-pane fade" id="tab-limits" role="tabpanel" aria-labelledby="tab-limits">
<div class="col-lg-10">
<table class="table table-bordered table-striped caption-top">
<caption>Task limits</caption>
<thead>
<tr>
<th>Task</th>
<th class="text-center">Rate limit</th>
<th class="text-center">Timeouts</th>
</tr>
</thead>
<tbody id="limits-table">
{% for taskname in worker.get('registered', []) %}
<tr>
<td>{{ taskname }}</td>
<td class="col-lg-2">
<div class="form-group">
<div class="input-group">
<input class="form-control form-control-sm" type="number">
<button class="btn btn-primary btn-sm mx-1" type="button"
id="task-rate-limit-{{taskname}}">Apply</button>
</div>
</div>
</td>
<td class="col-lg-2">
<div class="form-group">
<div class="input-group">
<input class="form-control form-control-sm" type="number">
<button class="btn btn-primary btn-sm mx-1" type="button"
id="task-timeout-soft-{{taskname}}">Soft</button>
<button class="btn btn-primary btn-sm mx-1" type="button"
id="task-timeout-hard-{{taskname}}">Hard</button>
</div>
</div>
</td>
</tr>
{% end %}
</tbody>
</table>
</div>
</div> <!-- end limits tab -->
<div class="tab-pane fade" id="tab-config" role="tabpanel" aria-labelledby="tab-config">
<div class="col-lg-8">
<table class="table table-bordered table-striped caption-top">
<caption>Configuration options</caption>
<tbody>
{% for name,value in sorted(worker.get('conf', {}).items()) %}
{% if value is not None %}
<tr>
<td><a
href="https://docs.celeryq.dev/en/latest/userguide/configuration.html#{{ name.lower().replace('_', '-') }}"
target="_blank">{{ name }}</a></td>
<td>{{ value }}</td>
</tr>
{% end %}
{% end %}
</tbody>
</table>
</div>
</div> <!-- end config tab -->
<div class="tab-pane fade" id="tab-system" role="tabpanel" aria-labelledby="tab-system">
<div class="col-lg-8">
<table class="table table-bordered table-striped caption-top">
<caption>System usage statistics</caption>
<tbody>
{% if isinstance(worker['stats'].get('rusage', None), dict) %}
{% for name, value in worker['stats']['rusage'].items() %}
<tr>
<td>{{ name }}</td>
<td>{{ value }}</td>
</tr>
{% end %}
{% end %}
</tbody>
</table>
</div>
</div> <!-- end system tab -->
{% if other %}
<div class="tab-pane fade" id="tab-other" role="tabpanel" aria-labelledby="tab-other">
<div class="col-lg-8">
<table class="table table-bordered table-striped caption-top">
<caption>Other statistics</caption>
<tbody>
{% for name, value in other.items() %}
<tr>
<td>{{ name }}</td>
<td>{{ value }}</td>
</tr>
{% end %}
</tbody>
</table>
</div>
</div> <!-- end other tab -->
{% end %}
</div>
</div>
</div>
</div>
{% end %}
@@ -0,0 +1,58 @@
{% extends "base.html" %}
{% block navbar %}
{% module Template("navbar.html", active_tab="workers")%}
{% end %}
{% block container %}
<div class="container-fluid mt-3">
<figure class="table-responsive">
<table id="workers-table" class="table table-bordered table-striped table-hover w-100">
<thead>
<tr>
<th>Worker</th>
<th class="text-center">Status</th>
<th class="text-center">Active</th>
<th class="text-center">Processed</th>
<th class="text-center">Failed</th>
<th class="text-center">Succeeded</th>
<th class="text-center">Retried</th>
<th class="text-center">Load Average</th>
</tr>
</thead>
<tbody>
{% for name, info in workers.items() %}
<tr id="{{ url_escape(name) }}">
<td>{{ name }}</td>
<td>{{ info.get('status', None) }}</td>
<td>{{ info.get('active', 0) or 0 }}</td>
<td>{{ info.get('task-received', 0) }}</td>
<td>{{ info.get('task-failed', 0) }}</td>
<td>{{ info.get('task-succeeded', 0) }}</td>
<td>{{ info.get('task-retried', 0) }}</td>
<td>{{ humanize(info.get('loadavg', 'N/A')) }}</td>
</tr>
{% end %}
</tbody>
<tfoot>
<tr>
<th>Total</th>
<th></th>
<th></th>
<th></th>
<th></th>
<th></th>
<th></th>
<th></th>
</tr>
</tfoot>
</table>
</figure>
</div>
{% end %}
{% block extra_scripts %}
<script type="text/javascript">
var autorefresh = {{ autorefresh }};
</script>
{% end %}
@@ -0,0 +1,65 @@
import os
from tornado.web import StaticFileHandler, url
from .api import control, tasks, workers
from .utils import gen_cookie_secret
from .views import auth, monitor
from .views.broker import BrokerView
from .views.error import NotFoundErrorHandler
from .views.tasks import TasksDataTable, TasksView, TaskView
from .views.workers import WorkersView, WorkerView
settings = dict(
template_path=os.path.join(os.path.dirname(__file__), "templates"),
static_path=os.path.join(os.path.dirname(__file__), "static"),
cookie_secret=gen_cookie_secret(),
static_url_prefix='/static/',
login_url='/login',
)
handlers = [
# App
url(r"/", WorkersView, name='main'),
url(r"/workers", WorkersView, name='workers'),
url(r"/worker/(.+)", WorkerView, name='worker'),
url(r"/task/(.+)", TaskView, name='task'),
url(r"/tasks", TasksView, name='tasks'),
url(r"/tasks/datatable", TasksDataTable),
url(r"/broker", BrokerView, name='broker'),
# Worker API
(r"/api/workers", workers.ListWorkers),
(r"/api/worker/shutdown/(.+)", control.WorkerShutDown),
(r"/api/worker/pool/restart/(.+)", control.WorkerPoolRestart),
(r"/api/worker/pool/grow/(.+)", control.WorkerPoolGrow),
(r"/api/worker/pool/shrink/(.+)", control.WorkerPoolShrink),
(r"/api/worker/pool/autoscale/(.+)", control.WorkerPoolAutoscale),
(r"/api/worker/queue/add-consumer/(.+)", control.WorkerQueueAddConsumer),
(r"/api/worker/queue/cancel-consumer/(.+)",
control.WorkerQueueCancelConsumer),
# Task API
(r"/api/tasks", tasks.ListTasks),
(r"/api/task/types", tasks.ListTaskTypes),
(r"/api/queues/length", tasks.GetQueueLengths),
(r"/api/task/info/(.*)", tasks.TaskInfo),
(r"/api/task/apply/(.+)", tasks.TaskApply),
(r"/api/task/async-apply/(.+)", tasks.TaskAsyncApply),
(r"/api/task/send-task/(.+)", tasks.TaskSend),
(r"/api/task/result/(.+)", tasks.TaskResult),
(r"/api/task/abort/(.+)", tasks.TaskAbort),
(r"/api/task/timeout/(.+)", control.TaskTimout),
(r"/api/task/rate-limit/(.+)", control.TaskRateLimit),
(r"/api/task/revoke/(.+)", control.TaskRevoke),
# Metrics
(r"/metrics", monitor.Metrics),
(r"/healthcheck", monitor.Healthcheck),
# Static
(r"/static/(.*)", StaticFileHandler,
{"path": settings['static_path']}),
# Auth
(r"/login", auth.LoginHandler),
# Error
(r".*", NotFoundErrorHandler),
]
@@ -0,0 +1,55 @@
import base64
import os.path
import uuid
from .. import __version__
def gen_cookie_secret():
return base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes)
def bugreport(app=None):
try:
import celery
import humanize
import tornado
app = app or celery.Celery()
# pylint: disable=consider-using-f-string
return 'flower -> flower:%s tornado:%s humanize:%s%s' % (
__version__,
tornado.version,
getattr(humanize, '__version__', None) or getattr(humanize, 'VERSION'),
app.bugreport()
)
except (ImportError, AttributeError) as e:
return f"Error when generating bug report: {e}. Have you installed correct versions of Flower's dependencies?"
def abs_path(path):
path = os.path.expanduser(path)
if not os.path.isabs(path):
cwd = os.environ.get('PWD') or os.getcwd()
path = os.path.join(cwd, path)
return path
def prepend_url(url, prefix):
return '/' + prefix.strip('/') + url
def strtobool(val):
"""Convert a string representation of truth to true (1) or false (0).
True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
'val' is anything else.
"""
val = val.lower()
if val in ('y', 'yes', 't', 'true', 'on', '1'):
return 1
if val in ('n', 'no', 'f', 'false', 'off', '0'):
return 0
raise ValueError(f"invalid truth value {val!r}")
@@ -0,0 +1,262 @@
import asyncio
import json
import logging
import numbers
import socket
import sys
from urllib.parse import quote, unquote, urljoin, urlparse
from tornado import httpclient, ioloop
try:
import redis
except ImportError:
redis = None
logger = logging.getLogger(__name__)
class BrokerBase:
def __init__(self, broker_url, *_, **__):
purl = urlparse(broker_url)
self.host = purl.hostname
self.port = purl.port
self.vhost = purl.path[1:]
username = purl.username
password = purl.password
self.username = unquote(username) if username else username
self.password = unquote(password) if password else password
async def queues(self, names):
raise NotImplementedError
class RabbitMQ(BrokerBase):
def __init__(self, broker_url, http_api, io_loop=None, **__):
super().__init__(broker_url)
self.io_loop = io_loop or ioloop.IOLoop.instance()
self.host = self.host or 'localhost'
self.port = self.port or 15672
self.vhost = quote(self.vhost, '') or '/' if self.vhost != '/' else self.vhost
self.username = self.username or 'guest'
self.password = self.password or 'guest'
if not http_api:
http_api = f"http://{self.username}:{self.password}@{self.host}:{self.port}/api/{self.vhost}"
try:
self.validate_http_api(http_api)
except ValueError:
logger.error("Invalid broker api url: %s", http_api)
self.http_api = http_api
async def queues(self, names):
url = urljoin(self.http_api, 'queues/' + self.vhost)
api_url = urlparse(self.http_api)
username = unquote(api_url.username or '') or self.username
password = unquote(api_url.password or '') or self.password
http_client = httpclient.AsyncHTTPClient()
try:
response = await http_client.fetch(
url, auth_username=username, auth_password=password,
connect_timeout=1.0, request_timeout=2.0,
validate_cert=False)
except (socket.error, httpclient.HTTPError) as e:
logger.error("RabbitMQ management API call failed: %s", e)
return []
finally:
http_client.close()
if response.code == 200:
info = json.loads(response.body.decode())
return [x for x in info if x['name'] in names]
response.rethrow()
@classmethod
def validate_http_api(cls, http_api):
url = urlparse(http_api)
if url.scheme not in ('http', 'https'):
raise ValueError(f"Invalid http api schema: {url.scheme}")
class RedisBase(BrokerBase):
DEFAULT_SEP = '\x06\x16'
DEFAULT_PRIORITY_STEPS = [0, 3, 6, 9]
def __init__(self, broker_url, *_, **kwargs):
super().__init__(broker_url)
self.redis = None
if not redis:
raise ImportError('redis library is required')
broker_options = kwargs.get('broker_options', {})
self.priority_steps = broker_options.get(
'priority_steps', self.DEFAULT_PRIORITY_STEPS)
self.sep = broker_options.get('sep', self.DEFAULT_SEP)
self.broker_prefix = broker_options.get('global_keyprefix', '')
def _q_for_pri(self, queue, pri):
if pri not in self.priority_steps:
raise ValueError('Priority not in priority steps')
# pylint: disable=consider-using-f-string
return '{0}{1}{2}'.format(*((queue, self.sep, pri) if pri else (queue, '', '')))
async def queues(self, names):
queue_stats = []
for name in names:
priority_names = [self.broker_prefix + self._q_for_pri(
name, pri) for pri in self.priority_steps]
queue_stats.append({
'name': name,
'messages': sum((self.redis.llen(x) for x in priority_names))
})
return queue_stats
class Redis(RedisBase):
def __init__(self, broker_url, *args, **kwargs):
super().__init__(broker_url, *args, **kwargs)
self.host = self.host or 'localhost'
self.port = self.port or 6379
self.vhost = self._prepare_virtual_host(self.vhost)
self.redis = self._get_redis_client()
def _prepare_virtual_host(self, vhost):
if not isinstance(vhost, numbers.Integral):
if not vhost or vhost == '/':
vhost = 0
elif vhost.startswith('/'):
vhost = vhost[1:]
try:
vhost = int(vhost)
except ValueError as exc:
raise ValueError(f'Database is int between 0 and limit - 1, not {vhost}') from exc
return vhost
def _get_redis_client_args(self):
return {
'host': self.host,
'port': self.port,
'db': self.vhost,
'username': self.username,
'password': self.password
}
def _get_redis_client(self):
return redis.Redis(**self._get_redis_client_args())
class RedisSentinel(RedisBase):
def __init__(self, broker_url, *args, **kwargs):
super().__init__(broker_url, *args, **kwargs)
broker_options = kwargs.get('broker_options', {})
self.host = self.host or 'localhost'
self.port = self.port or 26379
self.vhost = self._prepare_virtual_host(self.vhost)
self.master_name = self._prepare_master_name(broker_options)
self.redis = self._get_redis_client(broker_options)
def _prepare_virtual_host(self, vhost):
if not isinstance(vhost, numbers.Integral):
if not vhost or vhost == '/':
vhost = 0
elif vhost.startswith('/'):
vhost = vhost[1:]
try:
vhost = int(vhost)
except ValueError as exc:
raise ValueError('Database is int between 0 and limit - 1, not {vhost}') from exc
return vhost
def _prepare_master_name(self, broker_options):
try:
master_name = broker_options['master_name']
except KeyError as exc:
raise ValueError('master_name is required for Sentinel broker') from exc
return master_name
def _get_redis_client(self, broker_options):
connection_kwargs = {
'password': self.password,
'sentinel_kwargs': broker_options.get('sentinel_kwargs')
}
# get all sentinel hosts from Celery App config and use them to initialize Sentinel
sentinel = redis.sentinel.Sentinel(
[(self.host, self.port)], **connection_kwargs)
redis_client = sentinel.master_for(self.master_name)
return redis_client
class RedisSocket(RedisBase):
def __init__(self, broker_url, *args, **kwargs):
super().__init__(broker_url, *args, **kwargs)
self.redis = redis.Redis(unix_socket_path='/' + self.vhost,
password=self.password)
class RedisSsl(Redis):
"""
Redis SSL class offering connection to the broker over SSL.
This does not currently support SSL settings through the url, only through
the broker_use_ssl celery configuration.
"""
def __init__(self, broker_url, *args, **kwargs):
if 'broker_use_ssl' not in kwargs:
raise ValueError('rediss broker requires broker_use_ssl')
self.broker_use_ssl = kwargs.get('broker_use_ssl', {})
super().__init__(broker_url, *args, **kwargs)
def _get_redis_client_args(self):
client_args = super()._get_redis_client_args()
client_args['ssl'] = True
if isinstance(self.broker_use_ssl, dict):
client_args.update(self.broker_use_ssl)
return client_args
class Broker:
def __new__(cls, broker_url, *args, **kwargs):
scheme = urlparse(broker_url).scheme
if scheme == 'amqp':
return RabbitMQ(broker_url, *args, **kwargs)
if scheme == 'redis':
return Redis(broker_url, *args, **kwargs)
if scheme == 'rediss':
return RedisSsl(broker_url, *args, **kwargs)
if scheme == 'redis+socket':
return RedisSocket(broker_url, *args, **kwargs)
if scheme == 'sentinel':
return RedisSentinel(broker_url, *args, **kwargs)
raise NotImplementedError
async def queues(self, names):
raise NotImplementedError
async def main():
broker_url = sys.argv[1] if len(sys.argv) > 1 else 'amqp://'
queue_name = sys.argv[2] if len(sys.argv) > 2 else 'celery'
if len(sys.argv) > 3:
http_api = sys.argv[3]
else:
http_api = 'http://guest:guest@localhost:15672/api/'
broker = Broker(broker_url, http_api=http_api)
queues = await broker.queues([queue_name])
if queues:
print(queues)
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,90 @@
import re
from kombu.utils.encoding import safe_str
def parse_search_terms(raw_search_value):
search_regexp = r'(?:[^\s,"]|"(?:\\.|[^"])*")+' # splits by space, ignores space in quotes
if not raw_search_value:
return {}
parsed_search = {}
for query_part in re.findall(search_regexp, raw_search_value):
if not query_part:
continue
if query_part.startswith('result:'):
parsed_search['result'] = preprocess_search_value(query_part[len('result:'):])
elif query_part.startswith('args:'):
if 'args' not in parsed_search:
parsed_search['args'] = []
parsed_search['args'].append(preprocess_search_value(query_part[len('args:'):]))
elif query_part.startswith('kwargs:'):
if 'kwargs'not in parsed_search:
parsed_search['kwargs'] = {}
try:
key, value = [p.strip() for p in query_part[len('kwargs:'):].split('=')]
except ValueError:
continue
parsed_search['kwargs'][key] = preprocess_search_value(value)
elif query_part.startswith('state'):
if 'state' not in parsed_search:
parsed_search['state'] = []
parsed_search['state'].append(preprocess_search_value(query_part[len('state:'):]))
else:
parsed_search['any'] = preprocess_search_value(query_part)
return parsed_search
def satisfies_search_terms(task, search_terms):
any_value_search_term = search_terms.get('any')
result_search_term = search_terms.get('result')
args_search_terms = search_terms.get('args')
kwargs_search_terms = search_terms.get('kwargs')
state_search_terms = search_terms.get('state')
if not any([any_value_search_term, result_search_term, args_search_terms, kwargs_search_terms, state_search_terms]):
return True
terms = [
state_search_terms and task.state in state_search_terms,
any_value_search_term and any_value_search_term in '|'.join(
filter(None, [task.name, task.uuid, task.state,
task.worker.hostname if task.worker else None,
task.args, task.kwargs, safe_str(task.result)])),
result_search_term and task.result and result_search_term in task.result,
kwargs_search_terms and all(
stringified_dict_contains_value(k, v, task.kwargs) for k, v in kwargs_search_terms.items()
),
args_search_terms and task_args_contains_search_args(task.args, args_search_terms)
]
return any(terms)
def stringified_dict_contains_value(key, value, str_dict):
"""Checks if dict in for of string like "{'test': 5}" contains
key/value pair. This works faster, then creating actual dict
from string since this operation is called for each task in case
of kwargs search."""
if not str_dict:
return False
value = str(value)
try:
# + 3 for key right quote, one for colon and one for space
key_index = str_dict.index(key) + len(key) + 3
except ValueError:
return False
try:
comma_index = str_dict.index(',', key_index)
except ValueError:
# last value in dict
comma_index = str_dict.index('}', key_index)
return str(value) == str_dict[key_index:comma_index].strip('"\'')
def preprocess_search_value(raw_value):
return raw_value.strip('" ') if raw_value else ''
def task_args_contains_search_args(task_args, search_args):
if not task_args:
return False
return all(a in task_args for a in search_args)
@@ -0,0 +1,71 @@
import datetime
import time
from .search import parse_search_terms, satisfies_search_terms
# pylint: disable=too-many-branches,too-many-locals,too-many-arguments
def iter_tasks(events, limit=None, offset=0, type=None, worker=None, state=None,
sort_by=None, received_start=None, received_end=None,
started_start=None, started_end=None, search=None):
i = 0
tasks = events.state.tasks_by_timestamp()
if sort_by is not None:
tasks = sort_tasks(tasks, sort_by)
def convert(x):
return time.mktime(datetime.datetime.strptime(x, '%Y-%m-%d %H:%M').timetuple())
search_terms = parse_search_terms(search or {})
for uuid, task in tasks:
if type and task.name != type:
continue
if worker and task.worker and task.worker.hostname != worker:
continue
if state and task.state != state:
continue
if received_start and task.received and\
task.received < convert(received_start):
continue
if received_end and task.received and\
task.received > convert(received_end):
continue
if started_start and task.started and\
task.started < convert(started_start):
continue
if started_end and task.started and\
task.started > convert(started_end):
continue
if not satisfies_search_terms(task, search_terms):
continue
if i >= offset:
yield uuid, task
i += 1
if limit is not None:
if i == limit + offset:
break
sort_keys = {'name': str, 'state': str, 'received': float, 'started': float}
def sort_tasks(tasks, sort_by):
assert sort_by.lstrip('-') in sort_keys
reverse = False
if sort_by.startswith('-'):
sort_by = sort_by.lstrip('-')
reverse = True
for task in sorted(
tasks,
key=lambda x: getattr(x[1], sort_by) or sort_keys[sort_by](),
reverse=reverse):
yield task
def get_task_by_id(events, task_id):
return events.state.tasks.get(task_id)
def as_dict(task):
return task.as_dict()
@@ -0,0 +1,44 @@
import re
from datetime import datetime, timedelta
from celery import current_app
from humanize import naturaltime
from pytz import timezone, utc
KEYWORDS_UP = ('ssl', 'uri', 'url', 'uuid', 'eta')
KEYWORDS_DOWN = ('args', 'kwargs')
UUID_REGEX = re.compile(r'^[\w]{8}(-[\w]{4}){3}-[\w]{12}$')
def format_time(time, tz):
dt = datetime.fromtimestamp(time, tz=tz)
return dt.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
def humanize(obj, type=None, length=None):
if obj is None:
obj = ''
elif type and type.startswith('time'):
tz = type[len('time'):].lstrip('-')
tz = timezone(tz) if tz else getattr(current_app, 'timezone', '') or utc
obj = format_time(float(obj), tz) if obj else ''
elif type and type.startswith('natural-time'):
tz = type[len('natural-time'):].lstrip('-')
tz = timezone(tz) if tz else getattr(current_app, 'timezone', '') or utc
delta = datetime.now(tz) - datetime.fromtimestamp(float(obj), tz)
if delta < timedelta(days=1):
obj = naturaltime(delta)
else:
obj = format_time(float(obj), tz) if obj else ''
elif isinstance(obj, str) and not re.match(UUID_REGEX, obj):
obj = obj.replace('-', ' ').replace('_', ' ')
obj = re.sub('|'.join(KEYWORDS_UP),
lambda m: m.group(0).upper(), obj)
if obj and obj not in KEYWORDS_DOWN:
obj = obj[0].upper() + obj[1:]
elif isinstance(obj, list):
if all(isinstance(x, (int, float, str)) for x in obj):
obj = ', '.join(map(str, obj))
if length is not None and len(obj) > length:
obj = obj[:length - 4] + ' ...'
return obj
@@ -0,0 +1,136 @@
import re
import inspect
import traceback
import copy
import logging
import hmac
from base64 import b64decode
import tornado
from ..utils import template, bugreport, strtobool
logger = logging.getLogger(__name__)
class BaseHandler(tornado.web.RequestHandler):
def set_default_headers(self):
if not (self.application.options.basic_auth or self.application.options.auth):
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers",
"x-requested-with,access-control-allow-origin,authorization,content-type")
self.set_header('Access-Control-Allow-Methods',
' PUT, DELETE, OPTIONS, POST, GET, PATCH')
def options(self, *_, **__):
self.set_status(204)
self.finish()
def render(self, *args, **kwargs):
app_options = self.application.options
functions = inspect.getmembers(template, inspect.isfunction)
assert not set(map(lambda x: x[0], functions)) & set(kwargs.keys())
kwargs.update(functions)
kwargs.update(url_prefix=app_options.url_prefix)
super().render(*args, **kwargs)
def write_error(self, status_code, **kwargs):
if status_code in (404, 403):
message = ''
if 'exc_info' in kwargs and kwargs['exc_info'][0] == tornado.web.HTTPError:
message = kwargs['exc_info'][1].log_message
self.render('404.html', message=message)
elif status_code == 500:
error_trace = "".join(traceback.format_exception(*kwargs['exc_info']))
self.render('error.html',
debug=self.application.options.debug,
status_code=status_code,
error_trace=error_trace,
bugreport=bugreport())
elif status_code == 401:
self.set_status(status_code)
self.set_header('WWW-Authenticate', 'Basic realm="flower"')
self.finish('Access denied')
else:
message = ''
if 'exc_info' in kwargs and kwargs['exc_info'][0] == tornado.web.HTTPError:
message = kwargs['exc_info'][1].log_message
self.set_header('Content-Type', 'text/plain')
self.write(str(message))
self.set_status(status_code)
self.finish()
def get_current_user(self):
# Basic Auth
basic_auth = self.application.options.basic_auth
if basic_auth:
auth_header = self.request.headers.get("Authorization", "")
try:
basic, credentials = auth_header.split()
credentials = b64decode(credentials.encode()).decode()
if basic != 'Basic':
raise tornado.web.HTTPError(401)
for stored_credential in basic_auth:
if hmac.compare_digest(stored_credential, credentials):
break
else:
raise tornado.web.HTTPError(401)
except ValueError as exc:
raise tornado.web.HTTPError(401) from exc
# OAuth2
if not self.application.options.auth:
return True
user = self.get_secure_cookie('user')
if user:
if not isinstance(user, str):
user = user.decode()
if re.match(self.application.options.auth, user):
return user
return None
# pylint: disable=dangerous-default-value
def get_argument(self, name, default=[], strip=True, type=None):
arg = super().get_argument(name, default, strip)
if arg and isinstance(arg, str):
arg = tornado.escape.xhtml_escape(arg)
if type is not None:
try:
if type is bool:
arg = strtobool(str(arg))
else:
arg = type(arg)
except (ValueError, TypeError) as exc:
if arg is None and default is None:
return arg
raise tornado.web.HTTPError(
400,
f"Invalid argument '{arg}' of type '{type.__name__}'") from exc
return arg
@property
def capp(self):
"return Celery application object"
return self.application.capp
def format_task(self, task):
custom_format_task = self.application.options.format_task
if custom_format_task:
try:
task = custom_format_task(copy.copy(task))
except Exception:
logger.exception("Failed to format '%s' task", task.uuid)
return task
def get_active_queue_names(self):
queues = set([])
for _, info in self.application.workers.items():
for queue in info.get('active_queues', []):
queues.add(queue['name'])
if not queues:
queues = set([self.capp.conf.task_default_queue]) |\
{q.name for q in self.capp.conf.task_queues or [] if q.name}
return sorted(queues)
@@ -0,0 +1,351 @@
import json
import os
import re
import uuid
from urllib.parse import urlencode
import tornado.auth
import tornado.gen
import tornado.web
from celery.utils.imports import instantiate
from tornado.options import options
from ..views import BaseHandler
from ..views.error import NotFoundErrorHandler
# pylint: disable=invalid-name
def authenticate(pattern, email):
if '|' in pattern:
return email in pattern.split('|')
if '*' in pattern:
pattern = re.escape(pattern).replace(r'\.\*', r"[A-Za-z0-9!#$%&'*+/=?^_`{|}~.\-]*")
return re.fullmatch(pattern, email)
return pattern == email
def validate_auth_option(pattern):
if pattern.count('*') > 1:
return False
if '*' in pattern and '|' in pattern:
return False
if '*' in pattern.rsplit('@', 1)[-1]:
return False
return True
class GoogleAuth2LoginHandler(BaseHandler, tornado.auth.GoogleOAuth2Mixin):
_OAUTH_SETTINGS_KEY = 'oauth'
async def get(self):
redirect_uri = self.settings[self._OAUTH_SETTINGS_KEY]['redirect_uri']
if self.get_argument('code', False):
user = await self.get_authenticated_user(
redirect_uri=redirect_uri,
code=self.get_argument('code'),
)
await self._on_auth(user)
else:
self.authorize_redirect(
redirect_uri=redirect_uri,
client_id=self.settings[self._OAUTH_SETTINGS_KEY]['key'],
scope=['profile', 'email'],
response_type='code',
extra_params={'approval_prompt': ''}
)
async def _on_auth(self, user):
if not user:
raise tornado.web.HTTPError(403, 'Google auth failed')
access_token = user['access_token']
try:
response = await self.get_auth_http_client().fetch(
'https://www.googleapis.com/userinfo/v2/me',
headers={'Authorization': f'Bearer {access_token}'})
except Exception as e:
raise tornado.web.HTTPError(403, f'Google auth failed: {e}')
email = json.loads(response.body.decode('utf-8'))['email']
if not authenticate(self.application.options.auth, email):
message = f"Access denied to '{email}'. Please use another account or ask your admin to add your email to flower --auth."
raise tornado.web.HTTPError(403, message)
self.set_secure_cookie("user", str(email))
next_ = self.get_argument('next', self.application.options.url_prefix or '/')
if self.application.options.url_prefix and next_[0] != '/':
next_ = '/' + next_
self.redirect(next_)
class LoginHandler(BaseHandler):
def __new__(cls, *args, **kwargs):
return instantiate(options.auth_provider or NotFoundErrorHandler, *args, **kwargs)
class GithubLoginHandler(BaseHandler, tornado.auth.OAuth2Mixin):
_OAUTH_DOMAIN = os.getenv(
"FLOWER_GITHUB_OAUTH_DOMAIN", "github.com")
_OAUTH_AUTHORIZE_URL = f'https://{_OAUTH_DOMAIN}/login/oauth/authorize'
_OAUTH_ACCESS_TOKEN_URL = f'https://{_OAUTH_DOMAIN}/login/oauth/access_token'
_OAUTH_NO_CALLBACKS = False
_OAUTH_SETTINGS_KEY = 'oauth'
async def get_authenticated_user(self, redirect_uri, code):
body = urlencode({
"redirect_uri": redirect_uri,
"code": code,
"client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'],
"client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'],
"grant_type": "authorization_code",
})
response = await self.get_auth_http_client().fetch(
self._OAUTH_ACCESS_TOKEN_URL,
method="POST",
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'}, body=body)
if response.error:
raise tornado.auth.AuthError(f'OAuth authenticator error: {response}')
return json.loads(response.body.decode('utf-8'))
async def get(self):
redirect_uri = self.settings[self._OAUTH_SETTINGS_KEY]['redirect_uri']
if self.get_argument('code', False):
user = await self.get_authenticated_user(
redirect_uri=redirect_uri,
code=self.get_argument('code'),
)
await self._on_auth(user)
else:
self.authorize_redirect(
redirect_uri=redirect_uri,
client_id=self.settings[self._OAUTH_SETTINGS_KEY]['key'],
scope=['user:email'],
response_type='code',
extra_params={'approval_prompt': ''}
)
async def _on_auth(self, user):
if not user:
raise tornado.web.HTTPError(500, 'OAuth authentication failed')
access_token = user['access_token']
response = await self.get_auth_http_client().fetch(
f'https://api.{self._OAUTH_DOMAIN}/user/emails',
headers={'Authorization': 'token ' + access_token,
'User-agent': 'Tornado auth'})
emails = [email['email'].lower() for email in json.loads(response.body.decode('utf-8'))
if email['verified'] and authenticate(self.application.options.auth, email['email'])]
if not emails:
message = (
"Access denied. Please use another account or "
"ask your admin to add your email to flower --auth."
)
raise tornado.web.HTTPError(403, message)
self.set_secure_cookie("user", str(emails.pop()))
next_ = self.get_argument('next', self.application.options.url_prefix or '/')
if self.application.options.url_prefix and next_[0] != '/':
next_ = '/' + next_
self.redirect(next_)
class GitLabLoginHandler(BaseHandler, tornado.auth.OAuth2Mixin):
_OAUTH_GITLAB_DOMAIN = os.getenv(
"FLOWER_GITLAB_OAUTH_DOMAIN", "gitlab.com")
_OAUTH_AUTHORIZE_URL = f'https://{_OAUTH_GITLAB_DOMAIN}/oauth/authorize'
_OAUTH_ACCESS_TOKEN_URL = f'https://{_OAUTH_GITLAB_DOMAIN}/oauth/token'
_OAUTH_NO_CALLBACKS = False
async def get_authenticated_user(self, redirect_uri, code):
body = urlencode({
'redirect_uri': redirect_uri,
'code': code,
'client_id': self.settings['oauth']['key'],
'client_secret': self.settings['oauth']['secret'],
'grant_type': 'authorization_code',
})
response = await self.get_auth_http_client().fetch(
self._OAUTH_ACCESS_TOKEN_URL,
method='POST',
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'},
body=body
)
if response.error:
raise tornado.auth.AuthError(f'OAuth authenticator error: {response}')
return json.loads(response.body.decode('utf-8'))
async def get(self):
redirect_uri = self.settings['oauth']['redirect_uri']
if self.get_argument('code', False):
user = await self.get_authenticated_user(
redirect_uri=redirect_uri,
code=self.get_argument('code'),
)
await self._on_auth(user)
else:
self.authorize_redirect(
redirect_uri=redirect_uri,
client_id=self.settings['oauth']['key'],
scope=['read_api'],
response_type='code',
extra_params={'approval_prompt': ''},
)
async def _on_auth(self, user):
if not user:
raise tornado.web.HTTPError(500, 'OAuth authentication failed')
access_token = user['access_token']
allowed_groups = os.environ.get('FLOWER_GITLAB_AUTH_ALLOWED_GROUPS', '')
allowed_groups = [group.strip() for group in allowed_groups.split(',') if group]
# Check user email address against regexp
try:
response = await self.get_auth_http_client().fetch(
f'https://{self._OAUTH_GITLAB_DOMAIN}/api/v4/user',
headers={'Authorization': 'Bearer ' + access_token,
'User-agent': 'Tornado auth'}
)
except Exception as e:
raise tornado.web.HTTPError(403, f'GitLab auth failed: {e}')
user_email = json.loads(response.body.decode('utf-8'))['email']
email_allowed = authenticate(self.application.options.auth, user_email)
# Check user's groups against list of allowed groups
matching_groups = []
if allowed_groups:
min_access_level = os.environ.get('FLOWER_GITLAB_MIN_ACCESS_LEVEL', '20')
response = await self.get_auth_http_client().fetch(
f'https://{self._OAUTH_GITLAB_DOMAIN}/api/v4/groups?min_access_level={min_access_level}',
headers={
'Authorization': 'Bearer ' + access_token,
'User-agent': 'Tornado auth'
}
)
matching_groups = [
group['id']
for group in json.loads(response.body.decode('utf-8'))
if group['full_path'] in allowed_groups
]
if not email_allowed or (allowed_groups and len(matching_groups) == 0):
message = 'Access denied. Please use another account or contact your admin.'
raise tornado.web.HTTPError(403, message)
self.set_secure_cookie('user', str(user_email))
next_ = self.get_argument('next', self.application.options.url_prefix or '/')
if self.application.options.url_prefix and next_[0] != '/':
next_ = '/' + next_
self.redirect(next_)
class OktaLoginHandler(BaseHandler, tornado.auth.OAuth2Mixin):
_OAUTH_NO_CALLBACKS = False
_OAUTH_SETTINGS_KEY = 'oauth'
@property
def base_url(self):
return os.environ.get('FLOWER_OAUTH2_OKTA_BASE_URL')
@property
def _OAUTH_AUTHORIZE_URL(self):
return f"{self.base_url}/v1/authorize"
@property
def _OAUTH_ACCESS_TOKEN_URL(self):
return f"{self.base_url}/v1/token"
@property
def _OAUTH_USER_INFO_URL(self):
return f"{self.base_url}/v1/userinfo"
async def get_access_token(self, redirect_uri, code):
body = urlencode({
"redirect_uri": redirect_uri,
"code": code,
"client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'],
"client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'],
"grant_type": "authorization_code",
})
response = await self.get_auth_http_client().fetch(
self._OAUTH_ACCESS_TOKEN_URL,
method="POST",
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'}, body=body)
if response.error:
raise tornado.auth.AuthError(f'OAuth authenticator error: {response}')
return json.loads(response.body.decode('utf-8'))
async def get(self):
redirect_uri = self.settings[self._OAUTH_SETTINGS_KEY]['redirect_uri']
if self.get_argument('code', False):
expected_state = (self.get_secure_cookie('oauth_state') or b'').decode('utf-8')
returned_state = self.get_argument('state')
if returned_state is None or returned_state != expected_state:
raise tornado.auth.AuthError(
'OAuth authenticator error: State tokens do not match')
access_token_response = await self.get_access_token(
redirect_uri=redirect_uri,
code=self.get_argument('code'),
)
await self._on_auth(access_token_response)
else:
state = str(uuid.uuid4())
self.set_secure_cookie("oauth_state", state)
self.authorize_redirect(
redirect_uri=redirect_uri,
client_id=self.settings[self._OAUTH_SETTINGS_KEY]['key'],
scope=['openid email'],
response_type='code',
extra_params={'state': state}
)
async def _on_auth(self, access_token_response):
if not access_token_response:
raise tornado.web.HTTPError(500, 'OAuth authentication failed')
access_token = access_token_response['access_token']
response = await self.get_auth_http_client().fetch(
self._OAUTH_USER_INFO_URL,
headers={'Authorization': 'Bearer ' + access_token,
'User-agent': 'Tornado auth'})
decoded_body = json.loads(response.body.decode('utf-8'))
email = (decoded_body.get('email') or '').strip()
email_verified = (
decoded_body.get('email_verified') and
authenticate(self.application.options.auth, email)
)
if not email_verified:
message = (
"Access denied. Please use another account or "
"ask your admin to add your email to flower --auth."
)
raise tornado.web.HTTPError(403, message)
self.set_secure_cookie("user", str(email))
self.clear_cookie('oauth_state')
next_ = self.get_argument('next', self.application.options.url_prefix or '/')
if self.application.options.url_prefix and next_[0] != '/':
next_ = '/' + next_
self.redirect(next_)
@@ -0,0 +1,35 @@
import logging
from tornado import web
from ..utils.broker import Broker
from ..views import BaseHandler
logger = logging.getLogger(__name__)
class BrokerView(BaseHandler):
@web.authenticated
async def get(self):
app = self.application
http_api = None
if app.transport == 'amqp' and app.options.broker_api:
http_api = app.options.broker_api
try:
broker = Broker(app.capp.connection(connect_timeout=1.0).as_uri(include_password=True),
http_api=http_api, broker_options=self.capp.conf.broker_transport_options,
broker_use_ssl=self.capp.conf.broker_use_ssl)
except NotImplementedError as exc:
raise web.HTTPError(
404, f"'{app.transport}' broker is not supported") from exc
try:
queues = await broker.queues(self.get_active_queue_names())
except Exception as e:
logger.error("Unable to get queues: '%s'", e)
self.render("broker.html",
broker_url=app.capp.connection().as_uri(),
queues=queues)
@@ -0,0 +1,11 @@
import tornado.web
from ..views import BaseHandler
class NotFoundErrorHandler(BaseHandler):
def get(self):
raise tornado.web.HTTPError(404)
def post(self):
raise tornado.web.HTTPError(404)
@@ -0,0 +1,14 @@
import prometheus_client
from ..views import BaseHandler
class Metrics(BaseHandler):
async def get(self):
self.write(prometheus_client.generate_latest())
self.set_header("Content-Type", "text/plain")
class Healthcheck(BaseHandler):
async def get(self):
self.write("OK")
@@ -0,0 +1,124 @@
import copy
import logging
from functools import total_ordering
from tornado import web
from ..utils.tasks import as_dict, get_task_by_id, iter_tasks
from ..views import BaseHandler
logger = logging.getLogger(__name__)
class TaskView(BaseHandler):
@web.authenticated
def get(self, task_id):
task = get_task_by_id(self.application.events, task_id)
if task is None:
raise web.HTTPError(404, f"Unknown task '{task_id}'")
task = self.format_task(task)
self.render("task.html", task=task)
@total_ordering
class Comparable:
"""
Compare two objects, one or more of which may be None. If one of the
values is None, the other will be deemed greater.
"""
def __init__(self, value):
self.value = value
def __eq__(self, other):
return self.value == other.value
def __lt__(self, other):
try:
return self.value < other.value
except TypeError:
return self.value is None
class TasksDataTable(BaseHandler):
@web.authenticated
def get(self):
app = self.application
draw = self.get_argument('draw', type=int)
start = self.get_argument('start', type=int)
length = self.get_argument('length', type=int)
search = self.get_argument('search[value]', type=str)
column = self.get_argument('order[0][column]', type=int)
sort_by = self.get_argument(f'columns[{column}][data]', type=str)
sort_order = self.get_argument('order[0][dir]', type=str) == 'desc'
def key(item):
return Comparable(getattr(item[1], sort_by))
self.maybe_normalize_for_sort(app.events.state.tasks_by_timestamp(), sort_by)
sorted_tasks = sorted(
iter_tasks(app.events, search=search),
key=key,
reverse=sort_order
)
filtered_tasks = []
for task in sorted_tasks[start:start + length]:
task_dict = as_dict(self.format_task(task)[1])
if task_dict.get('worker'):
task_dict['worker'] = task_dict['worker'].hostname
filtered_tasks.append(task_dict)
self.write(dict(draw=draw, data=filtered_tasks,
recordsTotal=len(sorted_tasks),
recordsFiltered=len(sorted_tasks)))
@classmethod
def maybe_normalize_for_sort(cls, tasks, sort_by):
sort_keys = {'name': str, 'state': str, 'received': float, 'started': float, 'runtime': float}
if sort_by in sort_keys:
for _, task in tasks:
attr_value = getattr(task, sort_by, None)
if attr_value:
try:
setattr(task, sort_by, sort_keys[sort_by](attr_value))
except TypeError:
pass
@web.authenticated
def post(self):
return self.get()
def format_task(self, task):
uuid, args = task
custom_format_task = self.application.options.format_task
if custom_format_task:
try:
args = custom_format_task(copy.copy(args))
except Exception:
logger.exception("Failed to format '%s' task", uuid)
return uuid, args
class TasksView(BaseHandler):
@web.authenticated
def get(self):
app = self.application
capp = self.application.capp
time = 'natural-time' if app.options.natural_time else 'time'
if capp.conf.timezone:
time += '-' + str(capp.conf.timezone)
self.render(
"tasks.html",
tasks=[],
columns=app.options.tasks_columns,
time=time,
)
@@ -0,0 +1,95 @@
import logging
import time
from tornado import web
from ..options import options
from ..views import BaseHandler
logger = logging.getLogger(__name__)
class WorkerView(BaseHandler):
@web.authenticated
async def get(self, name):
try:
self.application.update_workers(workername=name)
except Exception as e:
logger.error(e)
worker = self.application.workers.get(name)
if worker is None:
raise web.HTTPError(404, f"Unknown worker '{name}'")
if 'stats' not in worker:
raise web.HTTPError(404, f"Unable to get stats for '{name}' worker")
self.render("worker.html", worker=dict(worker, name=name))
class WorkersView(BaseHandler):
@web.authenticated
async def get(self):
refresh = self.get_argument('refresh', default=False, type=bool)
json = self.get_argument('json', default=False, type=bool)
events = self.application.events.state
if refresh:
try:
self.application.update_workers()
except Exception as e:
logger.exception('Failed to update workers: %s', e)
workers = {}
for name, values in events.counter.items():
if name not in events.workers:
continue
worker = events.workers[name]
info = dict(values)
info.update(self._as_dict(worker))
info.update(status=worker.alive)
workers[name] = info
if options.purge_offline_workers is not None:
timestamp = int(time.time())
offline_workers = []
for name, info in workers.items():
if info.get('status', True):
continue
heartbeats = info.get('heartbeats', [])
last_heartbeat = int(max(heartbeats)) if heartbeats else None
if not last_heartbeat or timestamp - last_heartbeat > options.purge_offline_workers:
offline_workers.append(name)
for name in offline_workers:
workers.pop(name)
if json:
self.write(dict(data=list(workers.values())))
else:
self.render("workers.html",
workers=workers,
broker=self.application.capp.connection().as_uri(),
autorefresh=1 if self.application.options.auto_refresh else 0)
@classmethod
def _as_dict(cls, worker):
if hasattr(worker, '_fields'):
return dict((k, getattr(worker, k)) for k in worker._fields)
return cls._info(worker)
@classmethod
def _info(cls, worker):
_fields = ('hostname', 'pid', 'freq', 'heartbeats', 'clock',
'active', 'processed', 'loadavg', 'sw_ident',
'sw_ver', 'sw_sys')
def _keys():
for key in _fields:
value = getattr(worker, key, None)
if value is not None:
yield key, value
return dict(_keys())