uktrade / directory-client-core
1 2
from functools import wraps
2 2
import json
3 2
import logging
4 2
from urllib.parse import urlencode
5

6 2
import requests
7 2
from requests.exceptions import RequestException
8 2
from w3lib.url import canonicalize_url
9

10 2
from django.conf import settings
11

12 2
from directory_client_core.cache_control import ETagCacheControl
13

14

15 2
logger = logging.getLogger(__name__)
16

17

18 2
MESSAGE_CACHE_HIT = 'Fallback cache hit. Using cached content.'
19 2
MESSAGE_CACHE_MISS = 'Fallback cache miss. Cannot use any content.'
20 2
MESSAGE_NOT_FOUND = 'Resource not found.'
21

22

23 2
class ThrottlingFilter(logging.Filter):
24
    """
25
    Filters out records that have been seen within the past <period of time>
26
    thereby reducing noise.
27

28
    How this works:
29
        - with `cache.add` the entry is stored only if the key is not yet
30
          present in the cache
31
        - cache.add returns True if the entry is stored, otherwise False
32
        - these cache entries expire after <period of time>.
33

34
    Therefore `filter` returns True if the key hasn't been seen in the past
35
    <period of time>, and False if it has. The logger takes this to mean
36
    "don't log this"
37

38
    """
39

40 2
    def __init__(self, cache):
41 2
        self.cache = cache
42 2
        self.timeout_in_seconds = getattr(
43
            settings,
44
            'DIRECTORY_CLIENT_CORE_CACHE_LOG_THROTTLING_SECONDS',
45
            None
46
        ) or 60*60*24  # default 24 hours
47

48 2
    def create_cache_key(sef, record):
49 2
        return f'noise-{record.getMessage()}-{record.url}'
50

51 2
    def filter(self, record):
52 2
        key = self.create_cache_key(record)
53 2
        return self.cache.add(key, '', timeout=self.timeout_in_seconds)
54

55

56 2
class PopulateResponseMixin:
57

58 2
    @classmethod
59
    def from_response(cls, raw_response):
60 2
        response = cls()
61 2
        response.__setstate__(raw_response.__getstate__())
62 2
        return response
63

64

65 2
class LiveResponse(PopulateResponseMixin, requests.Response):
66 2
    pass
67

68

69 2
class FailureResponse(PopulateResponseMixin, requests.Response):
70 2
    pass
71

72

73 2
class CacheResponse(requests.Response):
74

75 2
    @classmethod
76
    def from_cached_content(cls, cached_content):
77 2
        response = cls()
78 2
        response.status_code = 200
79 2
        response._content = cached_content
80 2
        return response
81

82

83 2
def fallback(cache):
84
    """
85
    Caches content retrieved by the client, thus allowing the cached
86
    content to be used later if the live content cannot be retrieved.
87

88
    """
89

90 2
    log_filter = ThrottlingFilter(cache=cache)
91 2
    logger.filters = []
92 2
    logger.addFilter(log_filter)
93

94 2
    def get_cache_control(cached_content):
95 2
        if cached_content:
96 2
            parsed = json.loads(cached_content.decode())
97 2
            if 'etag' in parsed:
98 2
                return ETagCacheControl(f'"{parsed["etag"]}"')
99

100 2
    def closure(func):
101 2
        @wraps(func)
102 2
        def wrapper(client, url, params={}, *args, **kwargs):
103 2
            cache_key = canonicalize_url(url + '?' + urlencode(params))
104 2
            cached_content = cache.get(cache_key, {})
105 2
            try:
106 2
                response = func(
107
                    client,
108
                    url=url,
109
                    params=params,
110
                    cache_control=get_cache_control(cached_content),
111
                    *args,
112
                    **kwargs,
113
                )
114 2
            except RequestException:
115
                # Failed to create the request e.g., the remote server is down,
116
                # perhaps a timeout occurred, or even connection closed by
117
                # remote, etc.
118 2
                if cached_content:
119 2
                    logger.error(MESSAGE_CACHE_HIT, extra={'url': url})
120 2
                    return CacheResponse.from_cached_content(cached_content)
121
                else:
122 2
                    raise
123
            else:
124 2
                log_context = {'status_code': response.status_code, 'url': url}
125 2
                if response.status_code == 404:
126 2
                    logger.error(MESSAGE_NOT_FOUND, extra=log_context)
127 2
                    return LiveResponse.from_response(response)
128 2
                elif response.status_code == 304:
129 2
                    return CacheResponse.from_cached_content(cached_content)
130 2
                elif not response.ok:
131
                    # Successfully requested the content, but the response is
132
                    # not OK (e.g., 500, 403, etc)
133 2
                    if cached_content:
134 2
                        logger.error(MESSAGE_CACHE_HIT, extra=log_context)
135 2
                        return CacheResponse.from_cached_content(cached_content)
136
                    else:
137 2
                        logger.exception(MESSAGE_CACHE_MISS, extra=log_context)
138 2
                        return FailureResponse.from_response(response)
139
                else:
140 2
                    cache.set(
141
                        cache_key,
142
                        response.content,
143
                        settings.DIRECTORY_CLIENT_CORE_CACHE_EXPIRE_SECONDS
144
                    )
145 2
                    return LiveResponse.from_response(response)
146
            raise NotImplementedError('unreachable')
147 2
        return wrapper
148 2
    return closure

Read our documentation on viewing source code .

Loading