1
|
|
# Licensed under a 3-clause BSD style license - see LICENSE.rst
|
2
|
1
|
from __future__ import print_function
|
3
|
|
|
4
|
1
|
import warnings
|
5
|
1
|
import re
|
6
|
1
|
import time
|
7
|
1
|
from math import cos, radians
|
8
|
1
|
import requests
|
9
|
1
|
from bs4 import BeautifulSoup
|
10
|
1
|
from io import StringIO
|
11
|
|
|
12
|
1
|
from six import BytesIO
|
13
|
1
|
import astropy.units as u
|
14
|
1
|
import astropy.coordinates as coord
|
15
|
1
|
import astropy.io.votable as votable
|
16
|
|
|
17
|
1
|
from ..query import QueryWithLogin
|
18
|
1
|
from ..exceptions import InvalidQueryError, TimeoutError, NoResultsWarning
|
19
|
1
|
from ..utils import commons
|
20
|
1
|
from ..exceptions import TableParseError
|
21
|
|
|
22
|
1
|
__all__ = ['BaseWFAUClass', 'clean_catalog']
|
23
|
|
|
24
|
|
|
25
|
1
|
class BaseWFAUClass(QueryWithLogin):
|
26
|
|
|
27
|
|
"""
|
28
|
|
The BaseWFAUQuery class. This is intended to be inherited by other classes
|
29
|
|
that implement specific interfaces to Wide-Field Astronomy Unit
|
30
|
|
(http://www.roe.ac.uk/ifa/wfau/) archives
|
31
|
|
"""
|
32
|
1
|
BASE_URL = ""
|
33
|
1
|
LOGIN_URL = BASE_URL + "DBLogin"
|
34
|
1
|
IMAGE_URL = BASE_URL + "GetImage"
|
35
|
1
|
ARCHIVE_URL = BASE_URL + "ImageList"
|
36
|
1
|
REGION_URL = BASE_URL + "WSASQL"
|
37
|
1
|
CROSSID_URL = BASE_URL + "CrossID"
|
38
|
1
|
TIMEOUT = ""
|
39
|
|
|
40
|
1
|
def __init__(self, username=None, password=None, community=None,
|
41
|
|
database='', programme_id='all'):
|
42
|
|
"""
|
43
|
|
The BaseWFAUClass __init__ is meant to be overwritten
|
44
|
|
"""
|
45
|
1
|
super(BaseWFAUClass, self).__init__()
|
46
|
1
|
self.database = database
|
47
|
1
|
self.programme_id = programme_id
|
48
|
1
|
self.session = None
|
49
|
1
|
if username is None or password is None or community is None:
|
50
|
0
|
pass
|
51
|
|
else:
|
52
|
0
|
self.login(username, password, community)
|
53
|
|
|
54
|
1
|
def _login(self, username, password, community):
|
55
|
|
"""
|
56
|
|
Login to non-public data as a known user.
|
57
|
|
|
58
|
|
Parameters
|
59
|
|
----------
|
60
|
|
username : str
|
61
|
|
password : str
|
62
|
|
community : str
|
63
|
|
"""
|
64
|
|
|
65
|
|
# Construct cookie holder, URL opener, and retrieve login page
|
66
|
0
|
self.session = requests.session()
|
67
|
|
|
68
|
0
|
credentials = {'user': username, 'passwd': password,
|
69
|
|
'community': ' ', 'community2': community}
|
70
|
0
|
response = self.session.post(self.LOGIN_URL, data=credentials)
|
71
|
0
|
if not response.ok:
|
72
|
0
|
self.session = None
|
73
|
0
|
response.raise_for_status()
|
74
|
0
|
if 'FAILED to log in' in response.text:
|
75
|
0
|
self.session = None
|
76
|
0
|
raise Exception("Unable to log in with your given credentials.\n"
|
77
|
|
"Please try again.\n Note that you can continue "
|
78
|
|
"to access public data without logging in.\n")
|
79
|
|
|
80
|
1
|
def logged_in(self):
|
81
|
|
"""
|
82
|
|
Determine whether currently logged in.
|
83
|
|
"""
|
84
|
1
|
if self.session is None:
|
85
|
1
|
return False
|
86
|
0
|
for cookie in self.session.cookies:
|
87
|
0
|
if cookie.is_expired():
|
88
|
0
|
return False
|
89
|
0
|
return True
|
90
|
|
|
91
|
1
|
def _args_to_payload(self, *args, **kwargs):
|
92
|
1
|
request_payload = {}
|
93
|
|
|
94
|
1
|
request_payload['database'] = kwargs.get('database', self.database)
|
95
|
|
|
96
|
1
|
programme_id = kwargs.get('programme_id', self.programme_id)
|
97
|
|
|
98
|
1
|
request_payload['programmeID'] = self._verify_programme_id(
|
99
|
|
programme_id, query_type=kwargs['query_type'])
|
100
|
1
|
sys = self._parse_system(kwargs.get('system'))
|
101
|
1
|
request_payload['sys'] = sys
|
102
|
1
|
if sys == 'J':
|
103
|
1
|
C = commons.parse_coordinates(args[0]).transform_to(coord.ICRS)
|
104
|
1
|
request_payload['ra'] = C.ra.degree
|
105
|
1
|
request_payload['dec'] = C.dec.degree
|
106
|
0
|
elif sys == 'G':
|
107
|
0
|
C = commons.parse_coordinates(args[0]).transform_to(coord.Galactic)
|
108
|
0
|
request_payload['ra'] = C.l.degree
|
109
|
0
|
request_payload['dec'] = C.b.degree
|
110
|
1
|
return request_payload
|
111
|
|
|
112
|
1
|
def _verify_programme_id(self, pid, query_type='catalog'):
|
113
|
|
"""
|
114
|
|
Verify the programme ID is valid for the query being executed.
|
115
|
|
|
116
|
|
Parameters
|
117
|
|
----------
|
118
|
|
pid : int or str
|
119
|
|
The programme ID, either an integer (i.e., the # that will get passed
|
120
|
|
to the URL) or a string using the three-letter acronym for the
|
121
|
|
programme or its long name
|
122
|
|
|
123
|
|
Returns
|
124
|
|
-------
|
125
|
|
pid : int
|
126
|
|
Returns the integer version of the programme ID
|
127
|
|
|
128
|
|
Raises
|
129
|
|
------
|
130
|
|
ValueError
|
131
|
|
If the pid is 'all' and the query type is a catalog. You can query
|
132
|
|
all surveys for images, but not all catalogs.
|
133
|
|
"""
|
134
|
1
|
if pid == 'all' and query_type == 'image':
|
135
|
0
|
return 'all'
|
136
|
1
|
elif pid == 'all' and query_type == 'catalog':
|
137
|
0
|
raise ValueError(
|
138
|
|
"Cannot query all catalogs at once. Valid catalogs are: {0}.\n"
|
139
|
|
"Change programmeID to one of these."
|
140
|
|
.format(",".join(self.programmes_short.keys())))
|
141
|
1
|
elif pid in self.programmes_long:
|
142
|
0
|
return self.programmes_long[pid]
|
143
|
1
|
elif pid in self.programmes_short:
|
144
|
1
|
return self.programmes_short[pid]
|
145
|
0
|
elif query_type != 'image':
|
146
|
0
|
raise ValueError("programme_id {0} not recognized".format(pid))
|
147
|
|
|
148
|
1
|
def _parse_system(self, system):
|
149
|
1
|
if system is None:
|
150
|
1
|
return 'J'
|
151
|
1
|
elif system.lower() in ('g', 'gal', 'galactic'):
|
152
|
0
|
return 'G'
|
153
|
1
|
elif system.lower() in ('j', 'j2000', 'celestical', 'radec'):
|
154
|
1
|
return 'J'
|
155
|
|
|
156
|
1
|
def get_images(self, coordinates, waveband='all', frame_type='stack',
|
157
|
|
image_width=1 * u.arcmin, image_height=None, radius=None,
|
158
|
|
database=None, programme_id=None,
|
159
|
|
verbose=True, get_query_payload=False,
|
160
|
|
show_progress=True):
|
161
|
|
"""
|
162
|
|
Get an image around a target/ coordinates from a WFAU catalog.
|
163
|
|
|
164
|
|
Parameters
|
165
|
|
----------
|
166
|
|
coordinates : str or `astropy.coordinates` object
|
167
|
|
The target around which to search. It may be specified as a
|
168
|
|
string in which case it is resolved using online services or as
|
169
|
|
the appropriate `astropy.coordinates` object. ICRS coordinates
|
170
|
|
may also be entered as strings as specified in the
|
171
|
|
`astropy.coordinates` module.
|
172
|
|
waveband : str
|
173
|
|
The color filter to download. Must be one of ``'all'``, ``'J'``,
|
174
|
|
``'H'``, ``'K'``, ``'H2'``, ``'Z'``, ``'Y'``, ``'Br'``].
|
175
|
|
frame_type : str
|
176
|
|
The type of image. Must be one of ``'stack'``, ``'normal'``,
|
177
|
|
``'interleave'``, ``'deep_stack'``, ``'confidence'``,
|
178
|
|
``'difference'``, ``'leavstack'``, ``'all'``]
|
179
|
|
image_width : str or `~astropy.units.Quantity` object, optional
|
180
|
|
The image size (along X). Cannot exceed 15 arcmin. If missing,
|
181
|
|
defaults to 1 arcmin.
|
182
|
|
image_height : str or `~astropy.units.Quantity` object, optional
|
183
|
|
The image size (along Y). Cannot exceed 90 arcmin. If missing,
|
184
|
|
same as image_width.
|
185
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
186
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
187
|
|
appropriate `~astropy.units.Quantity` object from `astropy.units`
|
188
|
|
may also be used. When missing only image around the given position
|
189
|
|
rather than multi-frames are retrieved.
|
190
|
|
programme_id : str
|
191
|
|
The survey or programme in which to search for.
|
192
|
|
database : str
|
193
|
|
The WFAU database to use.
|
194
|
|
verbose : bool
|
195
|
|
Defaults to `True`. When `True` prints additional messages.
|
196
|
|
get_query_payload : bool, optional
|
197
|
|
If `True` then returns the dictionary sent as the HTTP request.
|
198
|
|
Defaults to `False`.
|
199
|
|
|
200
|
|
Returns
|
201
|
|
-------
|
202
|
|
list : A list of `~astropy.io.fits.HDUList` objects.
|
203
|
|
"""
|
204
|
|
|
205
|
1
|
readable_objs = self.get_images_async(
|
206
|
|
coordinates, waveband=waveband, frame_type=frame_type,
|
207
|
|
image_width=image_width, image_height=image_height,
|
208
|
|
database=database, programme_id=programme_id, radius=radius,
|
209
|
|
verbose=verbose, get_query_payload=get_query_payload,
|
210
|
|
show_progress=show_progress)
|
211
|
|
|
212
|
1
|
if get_query_payload:
|
213
|
0
|
return readable_objs
|
214
|
1
|
return [obj.get_fits() for obj in readable_objs]
|
215
|
|
|
216
|
1
|
def get_images_async(self, coordinates, waveband='all', frame_type='stack',
|
217
|
|
image_width=1 * u.arcmin, image_height=None,
|
218
|
|
radius=None, database=None,
|
219
|
|
programme_id=None, verbose=True,
|
220
|
|
get_query_payload=False,
|
221
|
|
show_progress=True):
|
222
|
|
"""
|
223
|
|
Serves the same purpose as
|
224
|
|
`~astroquery.wfau.BaseWFAUClass.get_images` but returns a list of
|
225
|
|
file handlers to remote files.
|
226
|
|
|
227
|
|
Parameters
|
228
|
|
----------
|
229
|
|
coordinates : str or `astropy.coordinates` object
|
230
|
|
The target around which to search. It may be specified as a
|
231
|
|
string in which case it is resolved using online services or as
|
232
|
|
the appropriate `astropy.coordinates` object. ICRS coordinates
|
233
|
|
may also be entered as strings as specified in the
|
234
|
|
`astropy.coordinates` module.
|
235
|
|
waveband : str
|
236
|
|
The color filter to download. Must be one of ``'all'``, ``'J'``,
|
237
|
|
``'H'``, ``'K'``, ``'H2'``, ``'Z'``, ``'Y'``, ``'Br'``].
|
238
|
|
frame_type : str
|
239
|
|
The type of image. Must be one of ``'stack'``, ``'normal'``,
|
240
|
|
``'interleave'``, ``'deep_stack'``, ``'confidence'``,
|
241
|
|
``'difference'``, ``'leavstack'``, ``'all'``]
|
242
|
|
image_width : str or `~astropy.units.Quantity` object, optional
|
243
|
|
The image size (along X). Cannot exceed 15 arcmin. If missing,
|
244
|
|
defaults to 1 arcmin.
|
245
|
|
image_height : str or `~astropy.units.Quantity` object, optional
|
246
|
|
The image size (along Y). Cannot exceed 90 arcmin. If missing,
|
247
|
|
same as image_width.
|
248
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
249
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
250
|
|
appropriate `~astropy.units.Quantity` object from `astropy.units`
|
251
|
|
may also be used. When missing only image around the given position
|
252
|
|
rather than multi-frames are retrieved.
|
253
|
|
programme_id : str
|
254
|
|
The survey or programme in which to search for. See
|
255
|
|
`list_catalogs`.
|
256
|
|
database : str
|
257
|
|
The WFAU database to use.
|
258
|
|
verbose : bool
|
259
|
|
Defaults to `True`. When `True` prints additional messages.
|
260
|
|
get_query_payload : bool, optional
|
261
|
|
If `True` then returns the dictionary sent as the HTTP request.
|
262
|
|
Defaults to `False`.
|
263
|
|
|
264
|
|
Returns
|
265
|
|
-------
|
266
|
|
list : list
|
267
|
|
A list of context-managers that yield readable file-like objects.
|
268
|
|
"""
|
269
|
|
|
270
|
1
|
if database is None:
|
271
|
1
|
database = self.database
|
272
|
|
|
273
|
1
|
if programme_id is None:
|
274
|
0
|
programme_id = self.programme_id
|
275
|
|
|
276
|
1
|
image_urls = self.get_image_list(coordinates, waveband=waveband,
|
277
|
|
frame_type=frame_type,
|
278
|
|
image_width=image_width,
|
279
|
|
image_height=image_height,
|
280
|
|
database=database, radius=radius,
|
281
|
|
programme_id=programme_id,
|
282
|
|
get_query_payload=get_query_payload)
|
283
|
1
|
if get_query_payload:
|
284
|
1
|
return image_urls
|
285
|
|
|
286
|
1
|
if verbose:
|
287
|
1
|
print("Found {num} targets".format(num=len(image_urls)))
|
288
|
|
|
289
|
1
|
return [commons.FileContainer(U, encoding='binary',
|
290
|
|
remote_timeout=self.TIMEOUT,
|
291
|
|
show_progress=show_progress)
|
292
|
|
for U in image_urls]
|
293
|
|
|
294
|
1
|
def get_image_list(self, coordinates, waveband='all', frame_type='stack',
|
295
|
|
image_width=1 * u.arcmin, image_height=None,
|
296
|
|
radius=None, database=None,
|
297
|
|
programme_id=None, get_query_payload=False):
|
298
|
|
"""
|
299
|
|
Function that returns a list of urls from which to download the FITS
|
300
|
|
images.
|
301
|
|
|
302
|
|
Parameters
|
303
|
|
----------
|
304
|
|
coordinates : str or `astropy.coordinates` object
|
305
|
|
The target around which to search. It may be specified as a
|
306
|
|
string in which case it is resolved using online services or as
|
307
|
|
the appropriate `astropy.coordinates` object. ICRS coordinates
|
308
|
|
may also be entered as strings as specified in the
|
309
|
|
`astropy.coordinates` module.
|
310
|
|
waveband : str
|
311
|
|
The color filter to download. Must be one of ``'all'``, ``'J'``,
|
312
|
|
``'H'``, ``'K'``, ``'H2'``, ``'Z'``, ``'Y'``, ``'Br'``].
|
313
|
|
frame_type : str
|
314
|
|
The type of image. Must be one of ``'stack'``, ``'normal'``,
|
315
|
|
``'interleave'``, ``'deep_stack'``, ``'confidence'``,
|
316
|
|
``'difference'``, ``'leavstack'``, ``'all'``]
|
317
|
|
image_width : str or `~astropy.units.Quantity` object, optional
|
318
|
|
The image size (along X). Cannot exceed 15 arcmin. If missing,
|
319
|
|
defaults to 1 arcmin.
|
320
|
|
image_height : str or `~astropy.units.Quantity` object, optional
|
321
|
|
The image size (along Y). Cannot exceed 90 arcmin. If missing,
|
322
|
|
same as image_width.
|
323
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
324
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
325
|
|
appropriate `~astropy.units.Quantity` object from
|
326
|
|
`astropy.units` may also be used. When missing only image around
|
327
|
|
the given position rather than multi-frames are retrieved.
|
328
|
|
programme_id : str
|
329
|
|
The survey or programme in which to search for. See
|
330
|
|
`list_catalogs`.
|
331
|
|
database : str
|
332
|
|
The WFAU database to use.
|
333
|
|
verbose : bool
|
334
|
|
Defaults to `True`. When `True` prints additional messages.
|
335
|
|
get_query_payload : bool, optional
|
336
|
|
If `True` then returns the dictionary sent as the HTTP request.
|
337
|
|
Defaults to `False`.
|
338
|
|
|
339
|
|
Returns
|
340
|
|
-------
|
341
|
|
url_list : list of image urls
|
342
|
|
|
343
|
|
"""
|
344
|
|
|
345
|
1
|
if frame_type not in self.frame_types:
|
346
|
0
|
raise ValueError("Invalid frame type. Valid frame types are: {!s}"
|
347
|
|
.format(self.frame_types))
|
348
|
|
|
349
|
1
|
if waveband not in self.filters:
|
350
|
0
|
raise ValueError("Invalid waveband. Valid wavebands are: {!s}"
|
351
|
|
.format(self.filters.keys()))
|
352
|
|
|
353
|
1
|
if database is None:
|
354
|
1
|
database = self.database
|
355
|
|
|
356
|
1
|
if programme_id is None:
|
357
|
0
|
programme_id = self.programme_id
|
358
|
|
|
359
|
1
|
request_payload = self._args_to_payload(coordinates, database=database,
|
360
|
|
programme_id=programme_id,
|
361
|
|
query_type='image')
|
362
|
1
|
request_payload['filterID'] = self.filters[waveband]
|
363
|
1
|
request_payload['obsType'] = 'object'
|
364
|
1
|
request_payload['frameType'] = self.frame_types[frame_type]
|
365
|
1
|
request_payload['mfid'] = ''
|
366
|
1
|
if radius is None:
|
367
|
1
|
request_payload['xsize'] = _parse_dimension(image_width)
|
368
|
1
|
if image_height is None:
|
369
|
1
|
request_payload['ysize'] = _parse_dimension(image_width)
|
370
|
|
else:
|
371
|
0
|
request_payload['ysize'] = _parse_dimension(image_height)
|
372
|
1
|
query_url = self.IMAGE_URL
|
373
|
|
else:
|
374
|
1
|
query_url = self.ARCHIVE_URL
|
375
|
1
|
ra = request_payload.pop('ra')
|
376
|
1
|
dec = request_payload.pop('dec')
|
377
|
1
|
radius = coord.Angle(radius).degree
|
378
|
1
|
del request_payload['sys']
|
379
|
1
|
request_payload['userSelect'] = 'default'
|
380
|
1
|
request_payload['minRA'] = str(
|
381
|
|
round(ra - radius / cos(radians(dec)), 2))
|
382
|
1
|
request_payload['maxRA'] = str(
|
383
|
|
round(ra + radius / cos(radians(dec)), 2))
|
384
|
1
|
request_payload['formatRA'] = 'degrees'
|
385
|
1
|
request_payload['minDec'] = str(dec - radius)
|
386
|
1
|
request_payload['maxDec'] = str(dec + radius)
|
387
|
1
|
request_payload['formatDec'] = 'degrees'
|
388
|
1
|
request_payload['startDay'] = 0
|
389
|
1
|
request_payload['startMonth'] = 0
|
390
|
1
|
request_payload['startYear'] = 0
|
391
|
1
|
request_payload['endDay'] = 0
|
392
|
1
|
request_payload['endMonth'] = 0
|
393
|
1
|
request_payload['endYear'] = 0
|
394
|
1
|
request_payload['dep'] = 0
|
395
|
1
|
request_payload['lmfid'] = ''
|
396
|
1
|
request_payload['fsid'] = ''
|
397
|
1
|
request_payload['rows'] = 1000
|
398
|
|
|
399
|
1
|
if get_query_payload:
|
400
|
1
|
return request_payload
|
401
|
|
|
402
|
1
|
response = self._wfau_send_request(query_url, request_payload)
|
403
|
1
|
response = self._check_page(response.url, "row")
|
404
|
|
|
405
|
1
|
image_urls = self.extract_urls(response.text)
|
406
|
|
# different links for radius queries and simple ones
|
407
|
1
|
if radius is not None:
|
408
|
1
|
image_urls = [link for link in image_urls if
|
409
|
|
('fits_download' in link and '_cat.fits'
|
410
|
|
not in link and '_two.fit' not in link)]
|
411
|
|
else:
|
412
|
1
|
image_urls = [link.replace("getImage", "getFImage")
|
413
|
|
for link in image_urls]
|
414
|
|
|
415
|
1
|
return image_urls
|
416
|
|
|
417
|
1
|
def extract_urls(self, html_in):
|
418
|
|
"""
|
419
|
|
Helper function that uses regexps to extract the image urls from the
|
420
|
|
given HTML.
|
421
|
|
|
422
|
|
Parameters
|
423
|
|
----------
|
424
|
|
html_in : str
|
425
|
|
source from which the urls are to be extracted.
|
426
|
|
|
427
|
|
Returns
|
428
|
|
-------
|
429
|
|
links : list
|
430
|
|
The list of URLS extracted from the input.
|
431
|
|
"""
|
432
|
|
# Parse html input for links
|
433
|
1
|
ahref = re.compile(r'href="([a-zA-Z0-9_\.&\?=%/:-]+)"')
|
434
|
1
|
links = ahref.findall(html_in)
|
435
|
1
|
return links
|
436
|
|
|
437
|
1
|
def query_region(self, coordinates, radius=1 * u.arcmin,
|
438
|
|
programme_id=None, database=None,
|
439
|
|
verbose=False, get_query_payload=False, system='J2000',
|
440
|
|
attributes=['default'], constraints=''):
|
441
|
|
"""
|
442
|
|
Used to query a region around a known identifier or given
|
443
|
|
coordinates from the catalog.
|
444
|
|
|
445
|
|
Parameters
|
446
|
|
----------
|
447
|
|
coordinates : str or `astropy.coordinates` object
|
448
|
|
The target around which to search. It may be specified as a string
|
449
|
|
in which case it is resolved using online services or as the
|
450
|
|
appropriate `astropy.coordinates` object. ICRS coordinates may also
|
451
|
|
be entered as strings as specified in the `astropy.coordinates`
|
452
|
|
module.
|
453
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
454
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
455
|
|
appropriate `~astropy.units.Quantity` object from
|
456
|
|
`astropy.units` may also be used. When missing defaults to 1
|
457
|
|
arcmin. Cannot exceed 90 arcmin.
|
458
|
|
programme_id : str
|
459
|
|
The survey or programme in which to search for. See
|
460
|
|
`list_catalogs`.
|
461
|
|
database : str
|
462
|
|
The WFAU database to use.
|
463
|
|
verbose : bool, optional.
|
464
|
|
When set to `True` displays warnings if the returned VOTable does
|
465
|
|
not conform to the standard. Defaults to `False`.
|
466
|
|
get_query_payload : bool, optional
|
467
|
|
If `True` then returns the dictionary sent as the HTTP request.
|
468
|
|
Defaults to `False`.
|
469
|
|
system : 'J2000' or 'Galactic'
|
470
|
|
The system in which to perform the query. Can affect the output
|
471
|
|
data columns.
|
472
|
|
attributes : list, optional.
|
473
|
|
Attributes to select from the table. See, e.g.,
|
474
|
|
http://horus.roe.ac.uk/vsa/crossID_notes.html
|
475
|
|
constraints : str, optional
|
476
|
|
SQL constraints to the search. Default is empty (no constrains
|
477
|
|
applied).
|
478
|
|
|
479
|
|
Returns
|
480
|
|
-------
|
481
|
|
result : `~astropy.table.Table`
|
482
|
|
Query result table.
|
483
|
|
"""
|
484
|
|
|
485
|
1
|
if database is None:
|
486
|
1
|
database = self.database
|
487
|
|
|
488
|
1
|
if programme_id is None:
|
489
|
0
|
if self.programme_id != 'all':
|
490
|
0
|
programme_id = self.programme_id
|
491
|
|
else:
|
492
|
0
|
raise ValueError("Must specify a programme_id for region queries")
|
493
|
|
|
494
|
1
|
response = self.query_region_async(coordinates, radius=radius,
|
495
|
|
programme_id=programme_id,
|
496
|
|
database=database,
|
497
|
|
get_query_payload=get_query_payload,
|
498
|
|
system=system, attributes=attributes,
|
499
|
|
constraints=constraints)
|
500
|
1
|
if get_query_payload:
|
501
|
0
|
return response
|
502
|
|
|
503
|
1
|
result = self._parse_result(response, verbose=verbose)
|
504
|
1
|
return result
|
505
|
|
|
506
|
1
|
def query_region_async(self, coordinates, radius=1 * u.arcmin,
|
507
|
|
programme_id=None,
|
508
|
|
database=None, get_query_payload=False,
|
509
|
|
system='J2000', attributes=['default'],
|
510
|
|
constraints=''):
|
511
|
|
"""
|
512
|
|
Serves the same purpose as `query_region`. But
|
513
|
|
returns the raw HTTP response rather than the parsed result.
|
514
|
|
|
515
|
|
Parameters
|
516
|
|
----------
|
517
|
|
coordinates : str or `astropy.coordinates` object
|
518
|
|
The target around which to search. It may be specified as a
|
519
|
|
string in which case it is resolved using online services or as
|
520
|
|
the appropriate `astropy.coordinates` object. ICRS coordinates
|
521
|
|
may also be entered as strings as specified in the
|
522
|
|
`astropy.coordinates` module.
|
523
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
524
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
525
|
|
appropriate `~astropy.units.Quantity` object from
|
526
|
|
`astropy.units` may also be used. When missing defaults to 1
|
527
|
|
arcmin. Cannot exceed 90 arcmin.
|
528
|
|
programme_id : str
|
529
|
|
The survey or programme in which to search for. See
|
530
|
|
`list_catalogs`.
|
531
|
|
database : str
|
532
|
|
The WFAU database to use.
|
533
|
|
get_query_payload : bool, optional
|
534
|
|
If `True` then returns the dictionary sent as the HTTP request.
|
535
|
|
Defaults to `False`.
|
536
|
|
attributes : list, optional.
|
537
|
|
Attributes to select from the table. See, e.g.,
|
538
|
|
http://horus.roe.ac.uk/vsa/crossID_notes.html
|
539
|
|
constraints : str, optional
|
540
|
|
SQL constraints to the search. Default is empty (no constrains
|
541
|
|
applied).
|
542
|
|
|
543
|
|
Returns
|
544
|
|
-------
|
545
|
|
response : `requests.Response`
|
546
|
|
The HTTP response returned from the service.
|
547
|
|
"""
|
548
|
|
|
549
|
1
|
if database is None:
|
550
|
1
|
database = self.database
|
551
|
|
|
552
|
1
|
if programme_id is None:
|
553
|
0
|
if self.programme_id != 'all':
|
554
|
0
|
programme_id = self.programme_id
|
555
|
|
else:
|
556
|
0
|
raise ValueError("Must specify a programme_id for region queries")
|
557
|
|
|
558
|
1
|
request_payload = self._args_to_payload(coordinates,
|
559
|
|
programme_id=programme_id,
|
560
|
|
database=database,
|
561
|
|
system=system,
|
562
|
|
query_type='catalog')
|
563
|
1
|
request_payload['radius'] = _parse_dimension(radius)
|
564
|
1
|
request_payload['from'] = 'source'
|
565
|
1
|
request_payload['formaction'] = 'region'
|
566
|
1
|
request_payload['xSize'] = ''
|
567
|
1
|
request_payload['ySize'] = ''
|
568
|
1
|
request_payload['boxAlignment'] = 'RADec'
|
569
|
1
|
request_payload['emailAddress'] = ''
|
570
|
1
|
request_payload['format'] = 'VOT'
|
571
|
1
|
request_payload['compress'] = 'NONE'
|
572
|
1
|
request_payload['rows'] = 1
|
573
|
1
|
request_payload['select'] = ','.join(attributes)
|
574
|
1
|
request_payload['where'] = constraints
|
575
|
|
|
576
|
|
# for some reason, this is required on the VISTA website
|
577
|
1
|
if self.archive is not None:
|
578
|
0
|
request_payload['archive'] = self.archive
|
579
|
|
|
580
|
1
|
if get_query_payload:
|
581
|
1
|
return request_payload
|
582
|
|
|
583
|
1
|
response = self._wfau_send_request(self.REGION_URL, request_payload)
|
584
|
1
|
response = self._check_page(response.url, "query finished")
|
585
|
|
|
586
|
1
|
return response
|
587
|
|
|
588
|
1
|
def _parse_result(self, response, verbose=False):
|
589
|
|
"""
|
590
|
|
Parses the raw HTTP response and returns it as a
|
591
|
|
`~astropy.table.Table`.
|
592
|
|
|
593
|
|
Parameters
|
594
|
|
----------
|
595
|
|
response : `requests.Response`
|
596
|
|
The HTTP response object
|
597
|
|
verbose : bool, optional
|
598
|
|
Defaults to `False`. If `True` it displays warnings whenever the
|
599
|
|
VOtable returned from the service doesn't conform to the standard.
|
600
|
|
|
601
|
|
Returns
|
602
|
|
-------
|
603
|
|
table : `~astropy.table.Table`
|
604
|
|
"""
|
605
|
1
|
table_links = self.extract_urls(response.text)
|
606
|
|
# keep only one link that is not a webstart
|
607
|
1
|
if len(table_links) == 0:
|
608
|
0
|
raise Exception("No VOTable found on returned webpage!")
|
609
|
1
|
table_link = [link for link in table_links if "8080" not in link][0]
|
610
|
1
|
with commons.get_readable_fileobj(table_link) as f:
|
611
|
1
|
content = f.read()
|
612
|
|
|
613
|
1
|
if not verbose:
|
614
|
1
|
commons.suppress_vo_warnings()
|
615
|
|
|
616
|
1
|
try:
|
617
|
1
|
io_obj = BytesIO(content.encode('utf-8'))
|
618
|
1
|
parsed_table = votable.parse(io_obj, pedantic=False)
|
619
|
1
|
first_table = parsed_table.get_first_table()
|
620
|
1
|
table = first_table.to_table()
|
621
|
1
|
if len(table) == 0:
|
622
|
0
|
warnings.warn("Query returned no results, so the table will "
|
623
|
|
"be empty", NoResultsWarning)
|
624
|
1
|
return table
|
625
|
0
|
except Exception as ex:
|
626
|
0
|
self.response = content
|
627
|
0
|
self.table_parse_error = ex
|
628
|
0
|
raise
|
629
|
0
|
raise TableParseError("Failed to parse WFAU votable! The raw "
|
630
|
|
"response can be found in self.response, "
|
631
|
|
"and the error in self.table_parse_error. "
|
632
|
|
"Exception: " + str(self.table_parse_error))
|
633
|
|
|
634
|
1
|
def list_catalogs(self, style='short'):
|
635
|
|
"""
|
636
|
|
Returns a list of available catalogs in WFAU.
|
637
|
|
These can be used as ``programme_id`` in queries.
|
638
|
|
|
639
|
|
Parameters
|
640
|
|
----------
|
641
|
|
style : str, optional
|
642
|
|
Must be one of ``'short'``, ``'long'``. Defaults to ``'short'``.
|
643
|
|
Determines whether to print long names or abbreviations for
|
644
|
|
catalogs.
|
645
|
|
|
646
|
|
Returns
|
647
|
|
-------
|
648
|
|
list : list containing catalog name strings in long or short style.
|
649
|
|
"""
|
650
|
0
|
if style == 'short':
|
651
|
0
|
return list(self.programmes_short.keys())
|
652
|
0
|
elif style == 'long':
|
653
|
0
|
return list(self.programmes_long.keys())
|
654
|
|
else:
|
655
|
0
|
warnings.warn("Style must be one of 'long', 'short'.\n"
|
656
|
|
"Returning catalog list in short format.\n")
|
657
|
0
|
return list(self.programmes_short.keys())
|
658
|
|
|
659
|
1
|
def _get_databases(self):
|
660
|
0
|
if self.logged_in():
|
661
|
0
|
response = self.session.get(url="/".join([self.BASE_URL,
|
662
|
|
self.IMAGE_FORM]))
|
663
|
|
else:
|
664
|
0
|
response = requests.get(url="/".join([self.BASE_URL,
|
665
|
|
self.IMAGE_FORM]))
|
666
|
|
|
667
|
0
|
root = BeautifulSoup(response.content, features='html5lib')
|
668
|
0
|
databases = [x.attrs['value'] for x in
|
669
|
|
root.find('select').findAll('option')]
|
670
|
0
|
return databases
|
671
|
|
|
672
|
1
|
def list_databases(self):
|
673
|
|
"""
|
674
|
|
List the databases available from the WFAU archive.
|
675
|
|
"""
|
676
|
0
|
self.databases = set(self.all_databases + tuple(self._get_databases()))
|
677
|
0
|
return self.databases
|
678
|
|
|
679
|
1
|
def _wfau_send_request(self, url, request_payload):
|
680
|
|
"""
|
681
|
|
Helper function that sends the query request via a session or simple
|
682
|
|
HTTP GET request.
|
683
|
|
|
684
|
|
Parameters
|
685
|
|
----------
|
686
|
|
url : str
|
687
|
|
The url to send the request to.
|
688
|
|
request_payload : dict
|
689
|
|
The dict of parameters for the GET request
|
690
|
|
|
691
|
|
Returns
|
692
|
|
-------
|
693
|
|
response : `requests.Response` object
|
694
|
|
The response for the HTTP GET request
|
695
|
|
"""
|
696
|
1
|
if hasattr(self, 'session') and self.logged_in():
|
697
|
0
|
response = self.session.get(url, params=request_payload,
|
698
|
|
timeout=self.TIMEOUT)
|
699
|
|
else:
|
700
|
1
|
response = self._request("GET", url=url, params=request_payload,
|
701
|
|
timeout=self.TIMEOUT)
|
702
|
1
|
return response
|
703
|
|
|
704
|
1
|
def _check_page(self, url, keyword, wait_time=1, max_attempts=30):
|
705
|
1
|
page_loaded = False
|
706
|
1
|
while not page_loaded and max_attempts > 0:
|
707
|
1
|
if self.logged_in():
|
708
|
0
|
response = self.session.get(url)
|
709
|
|
else:
|
710
|
1
|
response = requests.get(url=url)
|
711
|
1
|
self.response = response
|
712
|
1
|
content = response.text
|
713
|
1
|
if re.search("error", content, re.IGNORECASE):
|
714
|
1
|
raise InvalidQueryError(
|
715
|
|
"Service returned with an error! "
|
716
|
|
"Check self.response for more information.")
|
717
|
1
|
elif re.search(keyword, content, re.IGNORECASE):
|
718
|
1
|
page_loaded = True
|
719
|
1
|
max_attempts -= 1
|
720
|
|
# wait for wait_time seconds before checking again
|
721
|
1
|
time.sleep(wait_time)
|
722
|
1
|
if page_loaded is False:
|
723
|
0
|
raise TimeoutError("Page did not load.")
|
724
|
1
|
return response
|
725
|
|
|
726
|
1
|
def query_cross_id_async(self, coordinates, radius=1*u.arcsec,
|
727
|
|
programme_id=None, database=None, table="source",
|
728
|
|
constraints="", attributes='default',
|
729
|
|
pairing='all', system='J2000',
|
730
|
|
get_query_payload=False,
|
731
|
|
):
|
732
|
|
"""
|
733
|
|
Query the crossID server
|
734
|
|
|
735
|
|
Parameters
|
736
|
|
----------
|
737
|
|
coordinates : astropy.SkyCoord
|
738
|
|
An array of one or more astropy SkyCoord objects specifying the
|
739
|
|
objects to crossmatch against.
|
740
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
741
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
742
|
|
appropriate `~astropy.units.Quantity` object from
|
743
|
|
`astropy.units` may also be used. When missing defaults to 1
|
744
|
|
arcsec.
|
745
|
|
programme_id : str
|
746
|
|
The survey or programme in which to search for. See
|
747
|
|
`list_catalogs`.
|
748
|
|
database : str
|
749
|
|
The WFAU database to use.
|
750
|
|
table : str
|
751
|
|
The table ID, one of: "source", "detection", "synopticSource"
|
752
|
|
constraints : str
|
753
|
|
SQL constraints. If 'source' is selected, this will be expanded
|
754
|
|
automatically
|
755
|
|
attributes : str
|
756
|
|
Additional attributes to select from the table. See, e.g.,
|
757
|
|
http://horus.roe.ac.uk/vsa/crossID_notes.html
|
758
|
|
system : 'J2000' or 'Galactic'
|
759
|
|
The system in which to perform the query. Can affect the output
|
760
|
|
data columns.
|
761
|
|
get_query_payload : bool, optional
|
762
|
|
If `True` then returns the dictionary sent as the HTTP request.
|
763
|
|
Defaults to `False`.
|
764
|
|
"""
|
765
|
|
|
766
|
0
|
if table == "source":
|
767
|
0
|
constraints += "(priOrSec<=0 OR priOrSec=frameSetID)"
|
768
|
0
|
if database is None:
|
769
|
0
|
database = self.database
|
770
|
|
|
771
|
0
|
if programme_id is None:
|
772
|
0
|
if self.programme_id != 'all':
|
773
|
0
|
programme_id = self.programme_id
|
774
|
|
else:
|
775
|
0
|
raise ValueError("Must specify a programme_id")
|
776
|
|
|
777
|
0
|
request_payload = self._args_to_payload(coordinates,
|
778
|
|
programme_id=programme_id,
|
779
|
|
database=database,
|
780
|
|
system=system,
|
781
|
|
query_type='catalog')
|
782
|
0
|
request_payload['radius'] = _parse_dimension(radius)
|
783
|
0
|
request_payload['from'] = 'source'
|
784
|
0
|
request_payload['formaction'] = 'region'
|
785
|
0
|
request_payload['xSize'] = ''
|
786
|
0
|
request_payload['ySize'] = ''
|
787
|
0
|
request_payload['boxAlignment'] = 'RADec'
|
788
|
0
|
request_payload['emailAddress'] = ''
|
789
|
0
|
request_payload['format'] = 'VOT'
|
790
|
0
|
request_payload['compress'] = 'NONE'
|
791
|
0
|
request_payload['rows'] = 1
|
792
|
0
|
request_payload['select'] = 'default'
|
793
|
0
|
request_payload['where'] = ''
|
794
|
0
|
request_payload['disp'] = ''
|
795
|
0
|
request_payload['baseTable'] = table
|
796
|
0
|
request_payload['whereClause'] = constraints
|
797
|
0
|
request_payload['qType'] = 'form'
|
798
|
0
|
request_payload['selectList'] = attributes
|
799
|
0
|
request_payload['uploadFile'] = 'file.txt'
|
800
|
0
|
if pairing not in ('nearest', 'all'):
|
801
|
0
|
raise ValueError("pairing must be one of 'nearest' or 'all'")
|
802
|
0
|
request_payload['nearest'] = 0 if pairing == 'nearest' else 1
|
803
|
|
|
804
|
|
# for some reason, this is required on the VISTA website
|
805
|
0
|
if self.archive is not None:
|
806
|
0
|
request_payload['archive'] = self.archive
|
807
|
|
|
808
|
0
|
if get_query_payload:
|
809
|
0
|
return request_payload
|
810
|
|
|
811
|
0
|
fh = StringIO()
|
812
|
0
|
assert len(coordinates) > 0
|
813
|
0
|
for crd in coordinates:
|
814
|
0
|
fh.write("{0} {1}\n".format(crd.ra.deg, crd.dec.deg))
|
815
|
0
|
fh.seek(0)
|
816
|
|
|
817
|
0
|
if hasattr(self, 'session') and self.logged_in():
|
818
|
0
|
response = self.session.post(self.CROSSID_URL,
|
819
|
|
params=request_payload,
|
820
|
|
files={'file.txt': fh},
|
821
|
|
timeout=self.TIMEOUT)
|
822
|
|
else:
|
823
|
0
|
response = self._request("POST", url=self.CROSSID_URL,
|
824
|
|
params=request_payload,
|
825
|
|
files={'file.txt': fh},
|
826
|
|
timeout=self.TIMEOUT)
|
827
|
|
|
828
|
0
|
raise NotImplementedError("It appears we haven't implemented the file "
|
829
|
|
"upload correctly. Help is needed.")
|
830
|
|
|
831
|
|
# response = self._check_page(response.url, "query finished")
|
832
|
|
|
833
|
0
|
return response
|
834
|
|
|
835
|
1
|
def query_cross_id(self, *args, **kwargs):
|
836
|
|
"""
|
837
|
|
See `query_cross_id_async`
|
838
|
|
"""
|
839
|
0
|
get_query_payload = kwargs.get('get_query_payload', False)
|
840
|
0
|
verbose = kwargs.get('verbose', False)
|
841
|
|
|
842
|
0
|
response = self.query_cross_id_async(*args, **kwargs)
|
843
|
|
|
844
|
0
|
if get_query_payload:
|
845
|
0
|
return response
|
846
|
|
|
847
|
0
|
result = self._parse_result(response, verbose=verbose)
|
848
|
0
|
return result
|
849
|
|
|
850
|
|
|
851
|
1
|
def clean_catalog(wfau_catalog, clean_band='K_1', badclass=-9999,
|
852
|
|
maxerrbits=41, minerrbits=0, maxpperrbits=60):
|
853
|
|
"""
|
854
|
|
Attempt to remove 'bad' entries in a catalog.
|
855
|
|
|
856
|
|
Parameters
|
857
|
|
----------
|
858
|
|
wfau_catalog : `~astropy.io.fits.BinTableHDU`
|
859
|
|
A FITS binary table instance from the WFAU survey.
|
860
|
|
clean_band : ``'K_1'``, ``'K_2'``, ``'J'``, ``'H'``
|
861
|
|
The band to use for bad photometry flagging.
|
862
|
|
badclass : int
|
863
|
|
Class to exclude.
|
864
|
|
minerrbits : int
|
865
|
|
maxerrbits : int
|
866
|
|
Inside this range is the accepted number of error bits.
|
867
|
|
maxpperrbits : int
|
868
|
|
Exclude this type of error bit.
|
869
|
|
|
870
|
|
Examples
|
871
|
|
--------
|
872
|
|
"""
|
873
|
|
|
874
|
0
|
band = clean_band
|
875
|
0
|
mask = ((wfau_catalog[band + 'ERRBITS'] <= maxerrbits) *
|
876
|
|
(wfau_catalog[band + 'ERRBITS'] >= minerrbits) *
|
877
|
|
((wfau_catalog['PRIORSEC'] == wfau_catalog['FRAMESETID']) +
|
878
|
|
(wfau_catalog['PRIORSEC'] == 0)) *
|
879
|
|
(wfau_catalog[band + 'PPERRBITS'] < maxpperrbits)
|
880
|
|
)
|
881
|
0
|
if band + 'CLASS' in wfau_catalog.colnames:
|
882
|
0
|
mask *= (wfau_catalog[band + 'CLASS'] != badclass)
|
883
|
0
|
elif 'mergedClass' in wfau_catalog.colnames:
|
884
|
0
|
mask *= (wfau_catalog['mergedClass'] != badclass)
|
885
|
|
|
886
|
0
|
return wfau_catalog.data[mask]
|
887
|
|
|
888
|
|
|
889
|
1
|
def _parse_dimension(dim):
|
890
|
|
"""
|
891
|
|
Parses the radius and returns it in the format expected by WFAU.
|
892
|
|
|
893
|
|
Parameters
|
894
|
|
----------
|
895
|
|
dim : str, `~astropy.units.Quantity`
|
896
|
|
|
897
|
|
Returns
|
898
|
|
-------
|
899
|
|
dim_in_min : float
|
900
|
|
The value of the radius in arcminutes.
|
901
|
|
"""
|
902
|
1
|
if (isinstance(dim, u.Quantity) and
|
903
|
|
dim.unit in u.deg.find_equivalent_units()):
|
904
|
1
|
dim_in_min = dim.to(u.arcmin).value
|
905
|
|
# otherwise must be an Angle or be specified in hours...
|
906
|
|
else:
|
907
|
0
|
try:
|
908
|
0
|
new_dim = coord.Angle(dim).degree
|
909
|
0
|
dim_in_min = u.Quantity(
|
910
|
|
value=new_dim, unit=u.deg).to(u.arcmin).value
|
911
|
0
|
except (u.UnitsError, coord.errors.UnitsError, AttributeError):
|
912
|
0
|
raise u.UnitsError("Dimension not in proper units")
|
913
|
1
|
return dim_in_min
|