feat: initial commit - Phase 1 & 2 core features

backend/.venv/lib/python3.9/site-packages/flower/utils/__init__.py

@@ -0,0 +1,55 @@
+import base64
+import os.path
+import uuid
+
+from .. import __version__
+
+
+def gen_cookie_secret():
+    return base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes)
+
+
+def bugreport(app=None):
+    try:
+        import celery
+        import humanize
+        import tornado
+
+        app = app or celery.Celery()
+
+		# pylint: disable=consider-using-f-string
+        return 'flower   -> flower:%s tornado:%s humanize:%s%s' % (
+            __version__,
+            tornado.version,
+            getattr(humanize, '__version__', None) or getattr(humanize, 'VERSION'),
+            app.bugreport()
+        )
+    except (ImportError, AttributeError) as e:
+        return f"Error when generating bug report: {e}. Have you installed correct versions of Flower's dependencies?"
+
+
+def abs_path(path):
+    path = os.path.expanduser(path)
+    if not os.path.isabs(path):
+        cwd = os.environ.get('PWD') or os.getcwd()
+        path = os.path.join(cwd, path)
+    return path
+
+
+def prepend_url(url, prefix):
+    return '/' + prefix.strip('/') + url
+
+
+def strtobool(val):
+    """Convert a string representation of truth to true (1) or false (0).
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
+    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
+    'val' is anything else.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return 1
+    if val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return 0
+    raise ValueError(f"invalid truth value {val!r}")

backend/.venv/lib/python3.9/site-packages/flower/utils/broker.py

@@ -0,0 +1,262 @@
+import asyncio
+import json
+import logging
+import numbers
+import socket
+import sys
+from urllib.parse import quote, unquote, urljoin, urlparse
+
+from tornado import httpclient, ioloop
+
+try:
+    import redis
+except ImportError:
+    redis = None
+
+
+logger = logging.getLogger(__name__)
+
+
+class BrokerBase:
+    def __init__(self, broker_url, *_, **__):
+        purl = urlparse(broker_url)
+        self.host = purl.hostname
+        self.port = purl.port
+        self.vhost = purl.path[1:]
+
+        username = purl.username
+        password = purl.password
+
+        self.username = unquote(username) if username else username
+        self.password = unquote(password) if password else password
+
+    async def queues(self, names):
+        raise NotImplementedError
+
+
+class RabbitMQ(BrokerBase):
+    def __init__(self, broker_url, http_api, io_loop=None, **__):
+        super().__init__(broker_url)
+        self.io_loop = io_loop or ioloop.IOLoop.instance()
+
+        self.host = self.host or 'localhost'
+        self.port = self.port or 15672
+        self.vhost = quote(self.vhost, '') or '/' if self.vhost != '/' else self.vhost
+        self.username = self.username or 'guest'
+        self.password = self.password or 'guest'
+
+        if not http_api:
+            http_api = f"http://{self.username}:{self.password}@{self.host}:{self.port}/api/{self.vhost}"
+
+        try:
+            self.validate_http_api(http_api)
+        except ValueError:
+            logger.error("Invalid broker api url: %s", http_api)
+
+        self.http_api = http_api
+
+    async def queues(self, names):
+        url = urljoin(self.http_api, 'queues/' + self.vhost)
+        api_url = urlparse(self.http_api)
+        username = unquote(api_url.username or '') or self.username
+        password = unquote(api_url.password or '') or self.password
+
+        http_client = httpclient.AsyncHTTPClient()
+        try:
+            response = await http_client.fetch(
+                url, auth_username=username, auth_password=password,
+                connect_timeout=1.0, request_timeout=2.0,
+                validate_cert=False)
+        except (socket.error, httpclient.HTTPError) as e:
+            logger.error("RabbitMQ management API call failed: %s", e)
+            return []
+        finally:
+            http_client.close()
+
+        if response.code == 200:
+            info = json.loads(response.body.decode())
+            return [x for x in info if x['name'] in names]
+        response.rethrow()
+
+    @classmethod
+    def validate_http_api(cls, http_api):
+        url = urlparse(http_api)
+        if url.scheme not in ('http', 'https'):
+            raise ValueError(f"Invalid http api schema: {url.scheme}")
+
+
+class RedisBase(BrokerBase):
+    DEFAULT_SEP = '\x06\x16'
+    DEFAULT_PRIORITY_STEPS = [0, 3, 6, 9]
+
+    def __init__(self, broker_url, *_, **kwargs):
+        super().__init__(broker_url)
+        self.redis = None
+
+        if not redis:
+            raise ImportError('redis library is required')
+
+        broker_options = kwargs.get('broker_options', {})
+        self.priority_steps = broker_options.get(
+            'priority_steps', self.DEFAULT_PRIORITY_STEPS)
+        self.sep = broker_options.get('sep', self.DEFAULT_SEP)
+        self.broker_prefix = broker_options.get('global_keyprefix', '')
+
+    def _q_for_pri(self, queue, pri):
+        if pri not in self.priority_steps:
+            raise ValueError('Priority not in priority steps')
+        # pylint: disable=consider-using-f-string
+        return '{0}{1}{2}'.format(*((queue, self.sep, pri) if pri else (queue, '', '')))
+
+    async def queues(self, names):
+        queue_stats = []
+        for name in names:
+            priority_names = [self.broker_prefix + self._q_for_pri(
+                name, pri) for pri in self.priority_steps]
+            queue_stats.append({
+                'name': name,
+                'messages': sum((self.redis.llen(x) for x in priority_names))
+            })
+        return queue_stats
+
+
+class Redis(RedisBase):
+
+    def __init__(self, broker_url, *args, **kwargs):
+        super().__init__(broker_url, *args, **kwargs)
+        self.host = self.host or 'localhost'
+        self.port = self.port or 6379
+        self.vhost = self._prepare_virtual_host(self.vhost)
+        self.redis = self._get_redis_client()
+
+    def _prepare_virtual_host(self, vhost):
+        if not isinstance(vhost, numbers.Integral):
+            if not vhost or vhost == '/':
+                vhost = 0
+            elif vhost.startswith('/'):
+                vhost = vhost[1:]
+            try:
+                vhost = int(vhost)
+            except ValueError as exc:
+                raise ValueError(f'Database is int between 0 and limit - 1, not {vhost}') from exc
+        return vhost
+
+    def _get_redis_client_args(self):
+        return {
+            'host': self.host,
+            'port': self.port,
+            'db': self.vhost,
+            'username': self.username,
+            'password': self.password
+        }
+
+    def _get_redis_client(self):
+        return redis.Redis(**self._get_redis_client_args())
+
+
+class RedisSentinel(RedisBase):
+
+    def __init__(self, broker_url, *args, **kwargs):
+        super().__init__(broker_url, *args, **kwargs)
+        broker_options = kwargs.get('broker_options', {})
+        self.host = self.host or 'localhost'
+        self.port = self.port or 26379
+        self.vhost = self._prepare_virtual_host(self.vhost)
+        self.master_name = self._prepare_master_name(broker_options)
+        self.redis = self._get_redis_client(broker_options)
+
+    def _prepare_virtual_host(self, vhost):
+        if not isinstance(vhost, numbers.Integral):
+            if not vhost or vhost == '/':
+                vhost = 0
+            elif vhost.startswith('/'):
+                vhost = vhost[1:]
+            try:
+                vhost = int(vhost)
+            except ValueError as exc:
+                raise ValueError('Database is int between 0 and limit - 1, not {vhost}') from exc
+        return vhost
+
+    def _prepare_master_name(self, broker_options):
+        try:
+            master_name = broker_options['master_name']
+        except KeyError as exc:
+            raise ValueError('master_name is required for Sentinel broker') from exc
+        return master_name
+
+    def _get_redis_client(self, broker_options):
+        connection_kwargs = {
+            'password': self.password,
+            'sentinel_kwargs': broker_options.get('sentinel_kwargs')
+        }
+        # get all sentinel hosts from Celery App config and use them to initialize Sentinel
+        sentinel = redis.sentinel.Sentinel(
+            [(self.host, self.port)], **connection_kwargs)
+        redis_client = sentinel.master_for(self.master_name)
+        return redis_client
+
+
+class RedisSocket(RedisBase):
+
+    def __init__(self, broker_url, *args, **kwargs):
+        super().__init__(broker_url, *args, **kwargs)
+        self.redis = redis.Redis(unix_socket_path='/' + self.vhost,
+                                 password=self.password)
+
+
+class RedisSsl(Redis):
+    """
+    Redis SSL class offering connection to the broker over SSL.
+    This does not currently support SSL settings through the url, only through
+    the broker_use_ssl celery configuration.
+    """
+
+    def __init__(self, broker_url, *args, **kwargs):
+        if 'broker_use_ssl' not in kwargs:
+            raise ValueError('rediss broker requires broker_use_ssl')
+        self.broker_use_ssl = kwargs.get('broker_use_ssl', {})
+        super().__init__(broker_url, *args, **kwargs)
+
+    def _get_redis_client_args(self):
+        client_args = super()._get_redis_client_args()
+        client_args['ssl'] = True
+        if isinstance(self.broker_use_ssl, dict):
+            client_args.update(self.broker_use_ssl)
+        return client_args
+
+
+class Broker:
+    def __new__(cls, broker_url, *args, **kwargs):
+        scheme = urlparse(broker_url).scheme
+        if scheme == 'amqp':
+            return RabbitMQ(broker_url, *args, **kwargs)
+        if scheme == 'redis':
+            return Redis(broker_url, *args, **kwargs)
+        if scheme == 'rediss':
+            return RedisSsl(broker_url, *args, **kwargs)
+        if scheme == 'redis+socket':
+            return RedisSocket(broker_url, *args, **kwargs)
+        if scheme == 'sentinel':
+            return RedisSentinel(broker_url, *args, **kwargs)
+        raise NotImplementedError
+
+    async def queues(self, names):
+        raise NotImplementedError
+
+
+async def main():
+    broker_url = sys.argv[1] if len(sys.argv) > 1 else 'amqp://'
+    queue_name = sys.argv[2] if len(sys.argv) > 2 else 'celery'
+    if len(sys.argv) > 3:
+        http_api = sys.argv[3]
+    else:
+        http_api = 'http://guest:guest@localhost:15672/api/'
+
+    broker = Broker(broker_url, http_api=http_api)
+    queues = await broker.queues([queue_name])
+    if queues:
+        print(queues)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backend/.venv/lib/python3.9/site-packages/flower/utils/search.py

@@ -0,0 +1,90 @@
+import re
+
+from kombu.utils.encoding import safe_str
+
+
+def parse_search_terms(raw_search_value):
+    search_regexp = r'(?:[^\s,"]|"(?:\\.|[^"])*")+'  # splits by space, ignores space in quotes
+    if not raw_search_value:
+        return {}
+    parsed_search = {}
+    for query_part in re.findall(search_regexp, raw_search_value):
+        if not query_part:
+            continue
+        if query_part.startswith('result:'):
+            parsed_search['result'] = preprocess_search_value(query_part[len('result:'):])
+        elif query_part.startswith('args:'):
+            if 'args' not in parsed_search:
+                parsed_search['args'] = []
+            parsed_search['args'].append(preprocess_search_value(query_part[len('args:'):]))
+        elif query_part.startswith('kwargs:'):
+            if 'kwargs'not in parsed_search:
+                parsed_search['kwargs'] = {}
+            try:
+                key, value = [p.strip() for p in query_part[len('kwargs:'):].split('=')]
+            except ValueError:
+                continue
+            parsed_search['kwargs'][key] = preprocess_search_value(value)
+        elif query_part.startswith('state'):
+            if 'state' not in parsed_search:
+                parsed_search['state'] = []
+            parsed_search['state'].append(preprocess_search_value(query_part[len('state:'):]))
+        else:
+            parsed_search['any'] = preprocess_search_value(query_part)
+    return parsed_search
+
+
+def satisfies_search_terms(task, search_terms):
+    any_value_search_term = search_terms.get('any')
+    result_search_term = search_terms.get('result')
+    args_search_terms = search_terms.get('args')
+    kwargs_search_terms = search_terms.get('kwargs')
+    state_search_terms = search_terms.get('state')
+
+    if not any([any_value_search_term, result_search_term, args_search_terms, kwargs_search_terms, state_search_terms]):
+        return True
+
+    terms = [
+        state_search_terms and task.state in state_search_terms,
+        any_value_search_term and any_value_search_term in '|'.join(
+            filter(None, [task.name, task.uuid, task.state,
+                          task.worker.hostname if task.worker else None,
+                          task.args, task.kwargs, safe_str(task.result)])),
+        result_search_term and task.result and result_search_term in task.result,
+        kwargs_search_terms and all(
+            stringified_dict_contains_value(k, v, task.kwargs) for k, v in kwargs_search_terms.items()
+        ),
+        args_search_terms and task_args_contains_search_args(task.args, args_search_terms)
+    ]
+    return any(terms)
+
+
+def stringified_dict_contains_value(key, value, str_dict):
+    """Checks if dict in for of string like "{'test': 5}" contains
+    key/value pair. This works faster, then creating actual dict
+    from string since this operation is called for each task in case
+    of kwargs search."""
+    if not str_dict:
+        return False
+    value = str(value)
+    try:
+        # + 3 for key right quote, one for colon and one for space
+        key_index = str_dict.index(key) + len(key) + 3
+    except ValueError:
+        return False
+    try:
+        comma_index = str_dict.index(',', key_index)
+    except ValueError:
+        # last value in dict
+        comma_index = str_dict.index('}', key_index)
+    return str(value) == str_dict[key_index:comma_index].strip('"\'')
+
+
+def preprocess_search_value(raw_value):
+    return raw_value.strip('" ') if raw_value else ''
+
+
+def task_args_contains_search_args(task_args, search_args):
+    if not task_args:
+        return False
+    return all(a in task_args for a in search_args)

backend/.venv/lib/python3.9/site-packages/flower/utils/tasks.py

@@ -0,0 +1,71 @@
+import datetime
+import time
+
+from .search import parse_search_terms, satisfies_search_terms
+
+
+# pylint: disable=too-many-branches,too-many-locals,too-many-arguments
+def iter_tasks(events, limit=None, offset=0, type=None, worker=None, state=None,
+               sort_by=None, received_start=None, received_end=None,
+               started_start=None, started_end=None, search=None):
+    i = 0
+    tasks = events.state.tasks_by_timestamp()
+    if sort_by is not None:
+        tasks = sort_tasks(tasks, sort_by)
+
+    def convert(x):
+        return time.mktime(datetime.datetime.strptime(x, '%Y-%m-%d %H:%M').timetuple())
+
+    search_terms = parse_search_terms(search or {})
+
+    for uuid, task in tasks:
+        if type and task.name != type:
+            continue
+        if worker and task.worker and task.worker.hostname != worker:
+            continue
+        if state and task.state != state:
+            continue
+        if received_start and task.received and\
+                task.received < convert(received_start):
+            continue
+        if received_end and task.received and\
+                task.received > convert(received_end):
+            continue
+        if started_start and task.started and\
+                task.started < convert(started_start):
+            continue
+        if started_end and task.started and\
+                task.started > convert(started_end):
+            continue
+        if not satisfies_search_terms(task, search_terms):
+            continue
+        if i >= offset:
+            yield uuid, task
+        i += 1
+        if limit is not None:
+            if i == limit + offset:
+                break
+
+
+sort_keys = {'name': str, 'state': str, 'received': float, 'started': float}
+
+
+def sort_tasks(tasks, sort_by):
+    assert sort_by.lstrip('-') in sort_keys
+    reverse = False
+    if sort_by.startswith('-'):
+        sort_by = sort_by.lstrip('-')
+        reverse = True
+    for task in sorted(
+            tasks,
+            key=lambda x: getattr(x[1], sort_by) or sort_keys[sort_by](),
+            reverse=reverse):
+        yield task
+
+
+def get_task_by_id(events, task_id):
+    return events.state.tasks.get(task_id)
+
+
+def as_dict(task):
+    return task.as_dict()

backend/.venv/lib/python3.9/site-packages/flower/utils/template.py

@@ -0,0 +1,44 @@
+import re
+from datetime import datetime, timedelta
+
+from celery import current_app
+from humanize import naturaltime
+from pytz import timezone, utc
+
+KEYWORDS_UP = ('ssl', 'uri', 'url', 'uuid', 'eta')
+KEYWORDS_DOWN = ('args', 'kwargs')
+UUID_REGEX = re.compile(r'^[\w]{8}(-[\w]{4}){3}-[\w]{12}$')
+
+
+def format_time(time, tz):
+    dt = datetime.fromtimestamp(time, tz=tz)
+    return dt.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
+
+
+def humanize(obj, type=None, length=None):
+    if obj is None:
+        obj = ''
+    elif type and type.startswith('time'):
+        tz = type[len('time'):].lstrip('-')
+        tz = timezone(tz) if tz else getattr(current_app, 'timezone', '') or utc
+        obj = format_time(float(obj), tz) if obj else ''
+    elif type and type.startswith('natural-time'):
+        tz = type[len('natural-time'):].lstrip('-')
+        tz = timezone(tz) if tz else getattr(current_app, 'timezone', '') or utc
+        delta = datetime.now(tz) - datetime.fromtimestamp(float(obj), tz)
+        if delta < timedelta(days=1):
+            obj = naturaltime(delta)
+        else:
+            obj = format_time(float(obj), tz) if obj else ''
+    elif isinstance(obj, str) and not re.match(UUID_REGEX, obj):
+        obj = obj.replace('-', ' ').replace('_', ' ')
+        obj = re.sub('|'.join(KEYWORDS_UP),
+                     lambda m: m.group(0).upper(), obj)
+        if obj and obj not in KEYWORDS_DOWN:
+            obj = obj[0].upper() + obj[1:]
+    elif isinstance(obj, list):
+        if all(isinstance(x, (int, float, str)) for x in obj):
+            obj = ', '.join(map(str, obj))
+    if length is not None and len(obj) > length:
+        obj = obj[:length - 4] + ' ...'
+    return obj