test_helpers.py at 3ab49cf ⋅ uktrade/directory-client-core

import json
import logging
from unittest.mock import patch

from freezegun import freeze_time
import pytest
import requests_mock
import requests

from django.core.cache import caches

from directory_client_core.base import AbstractAPIClient
from directory_client_core import helpers


@pytest.fixture
def fallback_cache():
    return caches['fallback']


@pytest.fixture(autouse=True)
def clear_fallback_cache(fallback_cache):
    fallback_cache.clear()


@pytest.fixture
def cached_client(fallback_cache):

    class APIClient(AbstractAPIClient):
        version = 1

        @helpers.fallback(cache=fallback_cache)
        def get(self, *args, **kwargs):
            return super().get(*args, **kwargs)

        def retrieve(self, slug):
            return self.get(
                url='/some/path/{slug}/'.format(slug=slug),
                params={'x': 'y', 'a': 'b'},
            )

    return APIClient(
        base_url='http://example.com',
        api_key='debug',
        sender_id='test-sender',
        timeout=5,
    )


def test_good_response_cached(cached_client, fallback_cache):
    expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
    path = '/some/path/thing/'

    with requests_mock.mock() as mock:
        mock.get('http://example.com' + path, content=expected_data)
        cached_client.retrieve('thing')

    cache_key = path + '?a=b&x=y'
    assert fallback_cache.get(cache_key) == expected_data


def test_good_response_etag(cached_client, fallback_cache):
    expected_data = bytes(
        json.dumps({'key': 'value', 'etag': '123'}), 'utf8'
    )
    path = '/some/path/thing/'

    url = 'http://example.com' + path
    headers = {'ETag': '"123"'}

    # given the page has been cached
    with requests_mock.mock() as mock:
        mock.get(url, content=expected_data, headers=headers)
        cached_client.retrieve('thing')

    cache_key = path + '?a=b&x=y'
    assert fallback_cache.get(cache_key) == expected_data

    # when the same page is requested and the remote server returns 304
    with requests_mock.mock() as mock:
        mock.get(url, content=b'', headers=headers, status_code=304)
        response = cached_client.retrieve('thing')
        request = mock.request_history[0]

    # then the request exposed the etag cache headers
    assert request.headers['If-None-Match'] == '"123"'

    # and the cached content is returned
    assert isinstance(response, helpers.CacheResponse)


@freeze_time('2012-01-14')
def test_good_response_cache_timeout(cached_client, fallback_cache, settings):
    settings.DIRECTORY_CLIENT_CORE_CACHE_EXPIRE_SECONDS = 100
    expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
    path = '/some/path/thing/'

    with requests_mock.mock() as mock:
        mock.get('http://example.com' + path, content=expected_data)
        cached_client.retrieve('thing')

    cache_key = path + '?a=b&x=y'

    key = fallback_cache.make_key(cache_key)
    fallback_cache.validate_key(key)

    assert fallback_cache._expire_info.get(key) == 1326499300.0


def test_bad_resonse_cache_hit(cached_client, caplog):
    path = '/some/path/thing/'
    expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
    url = 'http://example.com' + path

    with requests_mock.mock() as mock:
        mock.get(url, content=expected_data)
        response_one = cached_client.retrieve('thing')

    with requests_mock.mock() as mock:
        mock.get(url, status_code=400)
        response_two = cached_client.retrieve('thing')

    assert response_one.status_code == 200
    assert response_one.content == expected_data
    assert isinstance(response_one, helpers.LiveResponse)

    assert response_two.status_code == 200
    assert response_two.content == expected_data
    assert isinstance(response_two, helpers.CacheResponse)

    log = caplog.records[-1]
    assert log.levelname == 'ERROR'
    assert log.msg == helpers.MESSAGE_CACHE_HIT
    assert log.status_code == 400
    assert log.url == path


def test_bad_response_cache_miss(cached_client, caplog):
    path = '/some/path/thing/'
    url = 'http://example.com' + path

    with requests_mock.mock() as mock:
        mock.get(url, status_code=400)
        response = cached_client.retrieve('thing')

    assert response.status_code == 400
    assert isinstance(response, helpers.FailureResponse)

    log = caplog.records[-1]
    assert log.levelname == 'ERROR'
    assert log.msg == helpers.MESSAGE_CACHE_MISS
    assert log.status_code == 400
    assert log.url == path


def test_bad_response_404(cached_client, caplog):
    path = '/some/path/thing/'
    url = 'http://example.com' + path

    with requests_mock.mock() as mock:
        mock.get(url, status_code=404)
        response = cached_client.retrieve('thing')

    assert response.status_code == 404
    assert isinstance(response, helpers.LiveResponse)

    log = caplog.records[-1]
    assert log.levelname == 'ERROR'
    assert log.msg == helpers.MESSAGE_NOT_FOUND
    assert log.status_code == 404
    assert log.url == path


def test_connection_error_cache_hit(cached_client, caplog):
    path = '/some/path/thing/'
    expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
    url = 'http://example.com' + path

    with requests_mock.mock() as mock:
        mock.get(url, content=expected_data)
        response_one = cached_client.retrieve('thing')

    with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
        mock_get.side_effect = requests.exceptions.ConnectionError()
        response_two = cached_client.retrieve('thing')

    assert response_one.status_code == 200
    assert response_one.content == expected_data
    assert isinstance(response_one, helpers.LiveResponse)

    assert response_two.status_code == 200
    assert response_two.content == expected_data
    assert isinstance(response_two, helpers.CacheResponse)

    log = caplog.records[-1]
    assert log.levelname == 'ERROR'
    assert log.msg == helpers.MESSAGE_CACHE_HIT
    assert log.url == path


def test_connection_error_cache_miss(cached_client, caplog):
    with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
        mock_get.side_effect = requests.exceptions.ConnectionError()

        with pytest.raises(requests.exceptions.ConnectionError):
            cached_client.retrieve('thing')

    assert len(caplog.records) == 0


def test_cache_querystrings(cached_client, fallback_cache):
    expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
    path = '/some/path/thing/'

    with requests_mock.mock() as mock:
        mock.get('http://example.com' + path, content=expected_data)
        cached_client.retrieve('thing',)

    cache_key = path + '?a=b&x=y'
    assert fallback_cache.get(cache_key) == expected_data


def test_logging_noise_filtering(cached_client, caplog):

    path = '/some/path/thing/'
    expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
    url = 'http://example.com' + path

    with requests_mock.mock() as mock:
        mock.get(url, content=expected_data)
        cached_client.retrieve('thing')

    with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
        mock_get.side_effect = requests.exceptions.ConnectionError()
        cached_client.retrieve('thing')

    errors = [item for item in caplog.records if item.levelname == 'ERROR']
    assert len(errors) == 1

    with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
        mock_get.side_effect = requests.exceptions.ConnectionError()
        cached_client.retrieve('thing')

    # second log entry was filtered out
    errors = [item for item in caplog.records if item.levelname == 'ERROR']
    assert len(errors) == 1


def test_throttling_log_filter_timeout(settings, fallback_cache):
    settings.DIRECTORY_CLIENT_CORE_CACHE_LOG_THROTTLING_SECONDS = 40
    log_filter = helpers.ThrottlingFilter(cache=fallback_cache)

    assert log_filter.timeout_in_seconds == 40


def test_throttling_log_filter_timeout_default_day(settings, fallback_cache):
    settings.DIRECTORY_CLIENT_CORE_CACHE_LOG_THROTTLING_SECONDS = None
    log_filter = helpers.ThrottlingFilter(cache=fallback_cache)

    assert log_filter.timeout_in_seconds == 60*60*24  # 24 hours


def test_throttling_filter(fallback_cache):
    log_filter = helpers.ThrottlingFilter(cache=fallback_cache)
    logger = logging.getLogger()
    record = logger.makeRecord(
        name='',
        level='ERROR',
        fn='',
        lno='',
        msg='something bad happened',
        args=[],
        exc_info='',
        extra={'url': 'https://www.google.com'}
    )

    assert log_filter.filter(record) is True
    assert log_filter.filter(record) is False

Read our documentation on viewing source code .

Navigation	Overlay
`t` Navigate files	`h` Toggle hits
`y` Change url to tip of branch	`m` Toggle misses
`b / v` Jump to prev/next hit line	`p` Toggle partial
`z / x` Jump to prev/next missed or partial line	`1..9` Toggle flags
`shift + o` Open current page in GitHub	`a` Toggle all on
`/ or ?` Show keyboard shortcuts dialog	`c` Toggle context lines or commits

1	2	import json
2	2	import logging
3	2	from unittest.mock import patch
4
5	2	from freezegun import freeze_time
6	2	import pytest
7	2	import requests_mock
8	2	import requests
9
10	2	from django.core.cache import caches
11
12	2	from directory_client_core.base import AbstractAPIClient
13	2	from directory_client_core import helpers
14
15
16	2	@pytest.fixture
17		def fallback_cache():
18	2	return caches['fallback']
19
20
21	2	@pytest.fixture(autouse=True)
22		def clear_fallback_cache(fallback_cache):
23	2	fallback_cache.clear()
24
25
26	2	@pytest.fixture
27		def cached_client(fallback_cache):
28
29	2	class APIClient(AbstractAPIClient):
30	2	version = 1
31
32	2	@helpers.fallback(cache=fallback_cache)
33		def get(self, args, *kwargs):
34	2	return super().get(args, *kwargs)
35
36	2	def retrieve(self, slug):
37	2	return self.get(
38		url='/some/path/{slug}/'.format(slug=slug),
39		params={'x': 'y', 'a': 'b'},
40		)
41
42	2	return APIClient(
43		base_url='http://example.com',
44		api_key='debug',
45		sender_id='test-sender',
46		timeout=5,
47		)
48
49
50	2	def test_good_response_cached(cached_client, fallback_cache):
51	2	expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
52	2	path = '/some/path/thing/'
53
54	2	with requests_mock.mock() as mock:
55	2	mock.get('http://example.com' + path, content=expected_data)
56	2	cached_client.retrieve('thing')
57
58	2	cache_key = path + '?a=b&x=y'
59	2	assert fallback_cache.get(cache_key) == expected_data
60
61
62	2	def test_good_response_etag(cached_client, fallback_cache):
63	2	expected_data = bytes(
64		json.dumps({'key': 'value', 'etag': '123'}), 'utf8'
65		)
66	2	path = '/some/path/thing/'
67
68	2	url = 'http://example.com' + path
69	2	headers = {'ETag': '"123"'}
70
71		# given the page has been cached
72	2	with requests_mock.mock() as mock:
73	2	mock.get(url, content=expected_data, headers=headers)
74	2	cached_client.retrieve('thing')
75
76	2	cache_key = path + '?a=b&x=y'
77	2	assert fallback_cache.get(cache_key) == expected_data
78
79		# when the same page is requested and the remote server returns 304
80	2	with requests_mock.mock() as mock:
81	2	mock.get(url, content=b'', headers=headers, status_code=304)
82	2	response = cached_client.retrieve('thing')
83	2	request = mock.request_history[0]
84
85		# then the request exposed the etag cache headers
86	2	assert request.headers['If-None-Match'] == '"123"'
87
88		# and the cached content is returned
89	2	assert isinstance(response, helpers.CacheResponse)
90
91
92	2	@freeze_time('2012-01-14')
93		def test_good_response_cache_timeout(cached_client, fallback_cache, settings):
94	2	settings.DIRECTORY_CLIENT_CORE_CACHE_EXPIRE_SECONDS = 100
95	2	expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
96	2	path = '/some/path/thing/'
97
98	2	with requests_mock.mock() as mock:
99	2	mock.get('http://example.com' + path, content=expected_data)
100	2	cached_client.retrieve('thing')
101
102	2	cache_key = path + '?a=b&x=y'
103
104	2	key = fallback_cache.make_key(cache_key)
105	2	fallback_cache.validate_key(key)
106
107	2	assert fallback_cache._expire_info.get(key) == 1326499300.0
108
109
110	2	def test_bad_resonse_cache_hit(cached_client, caplog):
111	2	path = '/some/path/thing/'
112	2	expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
113	2	url = 'http://example.com' + path
114
115	2	with requests_mock.mock() as mock:
116	2	mock.get(url, content=expected_data)
117	2	response_one = cached_client.retrieve('thing')
118
119	2	with requests_mock.mock() as mock:
120	2	mock.get(url, status_code=400)
121	2	response_two = cached_client.retrieve('thing')
122
123	2	assert response_one.status_code == 200
124	2	assert response_one.content == expected_data
125	2	assert isinstance(response_one, helpers.LiveResponse)
126
127	2	assert response_two.status_code == 200
128	2	assert response_two.content == expected_data
129	2	assert isinstance(response_two, helpers.CacheResponse)
130
131	2	log = caplog.records[-1]
132	2	assert log.levelname == 'ERROR'
133	2	assert log.msg == helpers.MESSAGE_CACHE_HIT
134	2	assert log.status_code == 400
135	2	assert log.url == path
136
137
138	2	def test_bad_response_cache_miss(cached_client, caplog):
139	2	path = '/some/path/thing/'
140	2	url = 'http://example.com' + path
141
142	2	with requests_mock.mock() as mock:
143	2	mock.get(url, status_code=400)
144	2	response = cached_client.retrieve('thing')
145
146	2	assert response.status_code == 400
147	2	assert isinstance(response, helpers.FailureResponse)
148
149	2	log = caplog.records[-1]
150	2	assert log.levelname == 'ERROR'
151	2	assert log.msg == helpers.MESSAGE_CACHE_MISS
152	2	assert log.status_code == 400
153	2	assert log.url == path
154
155
156	2	def test_bad_response_404(cached_client, caplog):
157	2	path = '/some/path/thing/'
158	2	url = 'http://example.com' + path
159
160	2	with requests_mock.mock() as mock:
161	2	mock.get(url, status_code=404)
162	2	response = cached_client.retrieve('thing')
163
164	2	assert response.status_code == 404
165	2	assert isinstance(response, helpers.LiveResponse)
166
167	2	log = caplog.records[-1]
168	2	assert log.levelname == 'ERROR'
169	2	assert log.msg == helpers.MESSAGE_NOT_FOUND
170	2	assert log.status_code == 404
171	2	assert log.url == path
172
173
174	2	def test_connection_error_cache_hit(cached_client, caplog):
175	2	path = '/some/path/thing/'
176	2	expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
177	2	url = 'http://example.com' + path
178
179	2	with requests_mock.mock() as mock:
180	2	mock.get(url, content=expected_data)
181	2	response_one = cached_client.retrieve('thing')
182
183	2	with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
184	2	mock_get.side_effect = requests.exceptions.ConnectionError()
185	2	response_two = cached_client.retrieve('thing')
186
187	2	assert response_one.status_code == 200
188	2	assert response_one.content == expected_data
189	2	assert isinstance(response_one, helpers.LiveResponse)
190
191	2	assert response_two.status_code == 200
192	2	assert response_two.content == expected_data
193	2	assert isinstance(response_two, helpers.CacheResponse)
194
195	2	log = caplog.records[-1]
196	2	assert log.levelname == 'ERROR'
197	2	assert log.msg == helpers.MESSAGE_CACHE_HIT
198	2	assert log.url == path
199
200
201	2	def test_connection_error_cache_miss(cached_client, caplog):
202	2	with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
203	2	mock_get.side_effect = requests.exceptions.ConnectionError()
204
205	2	with pytest.raises(requests.exceptions.ConnectionError):
206	2	cached_client.retrieve('thing')
207
208	2	assert len(caplog.records) == 0
209
210
211	2	def test_cache_querystrings(cached_client, fallback_cache):
212	2	expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
213	2	path = '/some/path/thing/'
214
215	2	with requests_mock.mock() as mock:
216	2	mock.get('http://example.com' + path, content=expected_data)
217	2	cached_client.retrieve('thing',)
218
219	2	cache_key = path + '?a=b&x=y'
220	2	assert fallback_cache.get(cache_key) == expected_data
221
222
223	2	def test_logging_noise_filtering(cached_client, caplog):
224
225	2	path = '/some/path/thing/'
226	2	expected_data = bytes(json.dumps({'key': 'value'}), 'utf8')
227	2	url = 'http://example.com' + path
228
229	2	with requests_mock.mock() as mock:
230	2	mock.get(url, content=expected_data)
231	2	cached_client.retrieve('thing')
232
233	2	with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
234	2	mock_get.side_effect = requests.exceptions.ConnectionError()
235	2	cached_client.retrieve('thing')
236
237	2	errors = [item for item in caplog.records if item.levelname == 'ERROR']
238	2	assert len(errors) == 1
239
240	2	with patch('directory_client_core.base.AbstractAPIClient.get') as mock_get:
241	2	mock_get.side_effect = requests.exceptions.ConnectionError()
242	2	cached_client.retrieve('thing')
243
244		# second log entry was filtered out
245	2	errors = [item for item in caplog.records if item.levelname == 'ERROR']
246	2	assert len(errors) == 1
247
248
249	2	def test_throttling_log_filter_timeout(settings, fallback_cache):
250	2	settings.DIRECTORY_CLIENT_CORE_CACHE_LOG_THROTTLING_SECONDS = 40
251	2	log_filter = helpers.ThrottlingFilter(cache=fallback_cache)
252
253	2	assert log_filter.timeout_in_seconds == 40
254
255
256	2	def test_throttling_log_filter_timeout_default_day(settings, fallback_cache):
257	2	settings.DIRECTORY_CLIENT_CORE_CACHE_LOG_THROTTLING_SECONDS = None
258	2	log_filter = helpers.ThrottlingFilter(cache=fallback_cache)
259
260	2	assert log_filter.timeout_in_seconds == 606024 # 24 hours
261
262
263	2	def test_throttling_filter(fallback_cache):
264	2	log_filter = helpers.ThrottlingFilter(cache=fallback_cache)
265	2	logger = logging.getLogger()
266	2	record = logger.makeRecord(
267		name='',
268		level='ERROR',
269		fn='',
270		lno='',
271		msg='something bad happened',
272		args=[],
273		exc_info='',
274		extra={'url': 'https://www.google.com'}
275		)
276
277	2	assert log_filter.filter(record) is True
278	2	assert log_filter.filter(record) is False