1
# Licensed under a 3-clause BSD style license - see LICENSE.rst
2 1
from __future__ import print_function
3

4 1
import warnings
5 1
import re
6 1
import time
7 1
from math import cos, radians
8 1
import requests
9 1
from bs4 import BeautifulSoup
10 1
from io import StringIO
11

12 1
from six import BytesIO
13 1
import astropy.units as u
14 1
import astropy.coordinates as coord
15 1
import astropy.io.votable as votable
16

17 1
from ..query import QueryWithLogin
18 1
from ..exceptions import InvalidQueryError, TimeoutError, NoResultsWarning
19 1
from ..utils import commons
20 1
from ..exceptions import TableParseError
21

22 1
__all__ = ['BaseWFAUClass', 'clean_catalog']
23

24

25 1
class BaseWFAUClass(QueryWithLogin):
26

27
    """
28
    The BaseWFAUQuery class.  This is intended to be inherited by other classes
29
    that implement specific interfaces to Wide-Field Astronomy Unit
30
    (http://www.roe.ac.uk/ifa/wfau/) archives
31
    """
32 1
    BASE_URL = ""
33 1
    LOGIN_URL = BASE_URL + "DBLogin"
34 1
    IMAGE_URL = BASE_URL + "GetImage"
35 1
    ARCHIVE_URL = BASE_URL + "ImageList"
36 1
    REGION_URL = BASE_URL + "WSASQL"
37 1
    CROSSID_URL = BASE_URL + "CrossID"
38 1
    TIMEOUT = ""
39

40 1
    def __init__(self, username=None, password=None, community=None,
41
                 database='', programme_id='all'):
42
        """
43
        The BaseWFAUClass __init__ is meant to be overwritten
44
        """
45 1
        super(BaseWFAUClass, self).__init__()
46 1
        self.database = database
47 1
        self.programme_id = programme_id
48 1
        self.session = None
49 1
        if username is None or password is None or community is None:
50 0
            pass
51
        else:
52 0
            self.login(username, password, community)
53

54 1
    def _login(self, username, password, community):
55
        """
56
        Login to non-public data as a known user.
57

58
        Parameters
59
        ----------
60
        username : str
61
        password : str
62
        community : str
63
        """
64

65
        # Construct cookie holder, URL opener, and retrieve login page
66 0
        self.session = requests.session()
67

68 0
        credentials = {'user': username, 'passwd': password,
69
                       'community': ' ', 'community2': community}
70 0
        response = self.session.post(self.LOGIN_URL, data=credentials)
71 0
        if not response.ok:
72 0
            self.session = None
73 0
            response.raise_for_status()
74 0
        if 'FAILED to log in' in response.text:
75 0
            self.session = None
76 0
            raise Exception("Unable to log in with your given credentials.\n"
77
                            "Please try again.\n Note that you can continue "
78
                            "to access public data without logging in.\n")
79

80 1
    def logged_in(self):
81
        """
82
        Determine whether currently logged in.
83
        """
84 1
        if self.session is None:
85 1
            return False
86 0
        for cookie in self.session.cookies:
87 0
            if cookie.is_expired():
88 0
                return False
89 0
        return True
90

91 1
    def _args_to_payload(self, *args, **kwargs):
92 1
        request_payload = {}
93

94 1
        request_payload['database'] = kwargs.get('database', self.database)
95

96 1
        programme_id = kwargs.get('programme_id', self.programme_id)
97

98 1
        request_payload['programmeID'] = self._verify_programme_id(
99
            programme_id, query_type=kwargs['query_type'])
100 1
        sys = self._parse_system(kwargs.get('system'))
101 1
        request_payload['sys'] = sys
102 1
        if sys == 'J':
103 1
            C = commons.parse_coordinates(args[0]).transform_to(coord.ICRS)
104 1
            request_payload['ra'] = C.ra.degree
105 1
            request_payload['dec'] = C.dec.degree
106 0
        elif sys == 'G':
107 0
            C = commons.parse_coordinates(args[0]).transform_to(coord.Galactic)
108 0
            request_payload['ra'] = C.l.degree
109 0
            request_payload['dec'] = C.b.degree
110 1
        return request_payload
111

112 1
    def _verify_programme_id(self, pid, query_type='catalog'):
113
        """
114
        Verify the programme ID is valid for the query being executed.
115

116
        Parameters
117
        ----------
118
        pid : int or str
119
            The programme ID, either an integer (i.e., the # that will get passed
120
            to the URL) or a string using the three-letter acronym for the
121
            programme or its long name
122

123
        Returns
124
        -------
125
        pid : int
126
            Returns the integer version of the programme ID
127

128
        Raises
129
        ------
130
        ValueError
131
            If the pid is 'all' and the query type is a catalog.  You can query
132
            all surveys for images, but not all catalogs.
133
        """
134 1
        if pid == 'all' and query_type == 'image':
135 0
            return 'all'
136 1
        elif pid == 'all' and query_type == 'catalog':
137 0
            raise ValueError(
138
                "Cannot query all catalogs at once. Valid catalogs are: {0}.\n"
139
                "Change programmeID to one of these."
140
                .format(",".join(self.programmes_short.keys())))
141 1
        elif pid in self.programmes_long:
142 0
            return self.programmes_long[pid]
143 1
        elif pid in self.programmes_short:
144 1
            return self.programmes_short[pid]
145 0
        elif query_type != 'image':
146 0
            raise ValueError("programme_id {0} not recognized".format(pid))
147

148 1
    def _parse_system(self, system):
149 1
        if system is None:
150 1
            return 'J'
151 1
        elif system.lower() in ('g', 'gal', 'galactic'):
152 0
            return 'G'
153 1
        elif system.lower() in ('j', 'j2000', 'celestical', 'radec'):
154 1
            return 'J'
155

156 1
    def get_images(self, coordinates, waveband='all', frame_type='stack',
157
                   image_width=1 * u.arcmin, image_height=None, radius=None,
158
                   database=None, programme_id=None,
159
                   verbose=True, get_query_payload=False,
160
                   show_progress=True):
161
        """
162
        Get an image around a target/ coordinates from a WFAU catalog.
163

164
        Parameters
165
        ----------
166
        coordinates : str or `astropy.coordinates` object
167
            The target around which to search. It may be specified as a
168
            string in which case it is resolved using online services or as
169
            the appropriate `astropy.coordinates` object. ICRS coordinates
170
            may also be entered as strings as specified in the
171
            `astropy.coordinates` module.
172
        waveband  : str
173
            The color filter to download. Must be one of  ``'all'``, ``'J'``,
174
            ``'H'``, ``'K'``, ``'H2'``, ``'Z'``, ``'Y'``, ``'Br'``].
175
        frame_type : str
176
            The type of image. Must be one of ``'stack'``, ``'normal'``,
177
            ``'interleave'``, ``'deep_stack'``, ``'confidence'``,
178
            ``'difference'``, ``'leavstack'``, ``'all'``]
179
        image_width : str or `~astropy.units.Quantity` object, optional
180
            The image size (along X). Cannot exceed 15 arcmin. If missing,
181
            defaults to 1 arcmin.
182
        image_height : str or `~astropy.units.Quantity` object, optional
183
             The image size (along Y). Cannot exceed 90 arcmin. If missing,
184
             same as image_width.
185
        radius : str or `~astropy.units.Quantity` object, optional
186
            The string must be parsable by `~astropy.coordinates.Angle`. The
187
            appropriate `~astropy.units.Quantity` object from `astropy.units`
188
            may also be used. When missing only image around the given position
189
            rather than multi-frames are retrieved.
190
        programme_id : str
191
            The survey or programme in which to search for.
192
        database : str
193
            The WFAU database to use.
194
        verbose : bool
195
            Defaults to `True`. When `True` prints additional messages.
196
        get_query_payload : bool, optional
197
            If `True` then returns the dictionary sent as the HTTP request.
198
            Defaults to `False`.
199

200
        Returns
201
        -------
202
        list : A list of `~astropy.io.fits.HDUList` objects.
203
        """
204

205 1
        readable_objs = self.get_images_async(
206
            coordinates, waveband=waveband, frame_type=frame_type,
207
            image_width=image_width, image_height=image_height,
208
            database=database, programme_id=programme_id, radius=radius,
209
            verbose=verbose, get_query_payload=get_query_payload,
210
            show_progress=show_progress)
211

212 1
        if get_query_payload:
213 0
            return readable_objs
214 1
        return [obj.get_fits() for obj in readable_objs]
215

216 1
    def get_images_async(self, coordinates, waveband='all', frame_type='stack',
217
                         image_width=1 * u.arcmin, image_height=None,
218
                         radius=None, database=None,
219
                         programme_id=None, verbose=True,
220
                         get_query_payload=False,
221
                         show_progress=True):
222
        """
223
        Serves the same purpose as
224
        `~astroquery.wfau.BaseWFAUClass.get_images` but returns a list of
225
        file handlers to remote files.
226

227
        Parameters
228
        ----------
229
        coordinates : str or `astropy.coordinates` object
230
            The target around which to search. It may be specified as a
231
            string in which case it is resolved using online services or as
232
            the appropriate `astropy.coordinates` object. ICRS coordinates
233
            may also be entered as strings as specified in the
234
            `astropy.coordinates` module.
235
        waveband  : str
236
            The color filter to download. Must be one of  ``'all'``, ``'J'``,
237
            ``'H'``, ``'K'``, ``'H2'``, ``'Z'``, ``'Y'``, ``'Br'``].
238
        frame_type : str
239
            The type of image. Must be one of ``'stack'``, ``'normal'``,
240
            ``'interleave'``, ``'deep_stack'``, ``'confidence'``,
241
            ``'difference'``, ``'leavstack'``, ``'all'``]
242
        image_width : str or `~astropy.units.Quantity` object, optional
243
            The image size (along X). Cannot exceed 15 arcmin. If missing,
244
            defaults to 1 arcmin.
245
        image_height : str or `~astropy.units.Quantity` object, optional
246
             The image size (along Y). Cannot exceed 90 arcmin. If missing,
247
             same as image_width.
248
        radius : str or `~astropy.units.Quantity` object, optional
249
            The string must be parsable by `~astropy.coordinates.Angle`. The
250
            appropriate `~astropy.units.Quantity` object from `astropy.units`
251
            may also be used. When missing only image around the given position
252
            rather than multi-frames are retrieved.
253
        programme_id : str
254
            The survey or programme in which to search for. See
255
            `list_catalogs`.
256
        database : str
257
            The WFAU database to use.
258
        verbose : bool
259
            Defaults to `True`. When `True` prints additional messages.
260
        get_query_payload : bool, optional
261
            If `True` then returns the dictionary sent as the HTTP request.
262
            Defaults to `False`.
263

264
        Returns
265
        -------
266
        list : list
267
            A list of context-managers that yield readable file-like objects.
268
        """
269

270 1
        if database is None:
271 1
            database = self.database
272

273 1
        if programme_id is None:
274 0
            programme_id = self.programme_id
275

276 1
        image_urls = self.get_image_list(coordinates, waveband=waveband,
277
                                         frame_type=frame_type,
278
                                         image_width=image_width,
279
                                         image_height=image_height,
280
                                         database=database, radius=radius,
281
                                         programme_id=programme_id,
282
                                         get_query_payload=get_query_payload)
283 1
        if get_query_payload:
284 1
            return image_urls
285

286 1
        if verbose:
287 1
            print("Found {num} targets".format(num=len(image_urls)))
288

289 1
        return [commons.FileContainer(U, encoding='binary',
290
                                      remote_timeout=self.TIMEOUT,
291
                                      show_progress=show_progress)
292
                for U in image_urls]
293

294 1
    def get_image_list(self, coordinates, waveband='all', frame_type='stack',
295
                       image_width=1 * u.arcmin, image_height=None,
296
                       radius=None, database=None,
297
                       programme_id=None, get_query_payload=False):
298
        """
299
        Function that returns a list of urls from which to download the FITS
300
        images.
301

302
        Parameters
303
        ----------
304
        coordinates : str or `astropy.coordinates` object
305
            The target around which to search. It may be specified as a
306
            string in which case it is resolved using online services or as
307
            the appropriate `astropy.coordinates` object. ICRS coordinates
308
            may also be entered as strings as specified in the
309
            `astropy.coordinates` module.
310
        waveband  : str
311
            The color filter to download. Must be one of  ``'all'``, ``'J'``,
312
            ``'H'``, ``'K'``, ``'H2'``, ``'Z'``, ``'Y'``, ``'Br'``].
313
        frame_type : str
314
            The type of image. Must be one of ``'stack'``, ``'normal'``,
315
            ``'interleave'``, ``'deep_stack'``, ``'confidence'``,
316
            ``'difference'``, ``'leavstack'``, ``'all'``]
317
        image_width : str or `~astropy.units.Quantity` object, optional
318
            The image size (along X). Cannot exceed 15 arcmin. If missing,
319
            defaults to 1 arcmin.
320
        image_height : str or `~astropy.units.Quantity` object, optional
321
             The image size (along Y). Cannot exceed 90 arcmin. If missing,
322
             same as image_width.
323
        radius : str or `~astropy.units.Quantity` object, optional
324
            The string must be parsable by `~astropy.coordinates.Angle`. The
325
            appropriate `~astropy.units.Quantity` object from
326
            `astropy.units` may also be used. When missing only image around
327
            the given position rather than multi-frames are retrieved.
328
        programme_id : str
329
            The survey or programme in which to search for. See
330
            `list_catalogs`.
331
        database : str
332
            The WFAU database to use.
333
        verbose : bool
334
            Defaults to `True`. When `True` prints additional messages.
335
        get_query_payload : bool, optional
336
            If `True` then returns the dictionary sent as the HTTP request.
337
            Defaults to `False`.
338

339
        Returns
340
        -------
341
        url_list : list of image urls
342

343
        """
344

345 1
        if frame_type not in self.frame_types:
346 0
            raise ValueError("Invalid frame type. Valid frame types are: {!s}"
347
                             .format(self.frame_types))
348

349 1
        if waveband not in self.filters:
350 0
            raise ValueError("Invalid waveband. Valid wavebands are: {!s}"
351
                             .format(self.filters.keys()))
352

353 1
        if database is None:
354 1
            database = self.database
355

356 1
        if programme_id is None:
357 0
            programme_id = self.programme_id
358

359 1
        request_payload = self._args_to_payload(coordinates, database=database,
360
                                                programme_id=programme_id,
361
                                                query_type='image')
362 1
        request_payload['filterID'] = self.filters[waveband]
363 1
        request_payload['obsType'] = 'object'
364 1
        request_payload['frameType'] = self.frame_types[frame_type]
365 1
        request_payload['mfid'] = ''
366 1
        if radius is None:
367 1
            request_payload['xsize'] = _parse_dimension(image_width)
368 1
            if image_height is None:
369 1
                request_payload['ysize'] = _parse_dimension(image_width)
370
            else:
371 0
                request_payload['ysize'] = _parse_dimension(image_height)
372 1
            query_url = self.IMAGE_URL
373
        else:
374 1
            query_url = self.ARCHIVE_URL
375 1
            ra = request_payload.pop('ra')
376 1
            dec = request_payload.pop('dec')
377 1
            radius = coord.Angle(radius).degree
378 1
            del request_payload['sys']
379 1
            request_payload['userSelect'] = 'default'
380 1
            request_payload['minRA'] = str(
381
                round(ra - radius / cos(radians(dec)), 2))
382 1
            request_payload['maxRA'] = str(
383
                round(ra + radius / cos(radians(dec)), 2))
384 1
            request_payload['formatRA'] = 'degrees'
385 1
            request_payload['minDec'] = str(dec - radius)
386 1
            request_payload['maxDec'] = str(dec + radius)
387 1
            request_payload['formatDec'] = 'degrees'
388 1
            request_payload['startDay'] = 0
389 1
            request_payload['startMonth'] = 0
390 1
            request_payload['startYear'] = 0
391 1
            request_payload['endDay'] = 0
392 1
            request_payload['endMonth'] = 0
393 1
            request_payload['endYear'] = 0
394 1
            request_payload['dep'] = 0
395 1
            request_payload['lmfid'] = ''
396 1
            request_payload['fsid'] = ''
397 1
            request_payload['rows'] = 1000
398

399 1
        if get_query_payload:
400 1
            return request_payload
401

402 1
        response = self._wfau_send_request(query_url, request_payload)
403 1
        response = self._check_page(response.url, "row")
404

405 1
        image_urls = self.extract_urls(response.text)
406
        # different links for radius queries and simple ones
407 1
        if radius is not None:
408 1
            image_urls = [link for link in image_urls if
409
                          ('fits_download' in link and '_cat.fits'
410
                           not in link and '_two.fit' not in link)]
411
        else:
412 1
            image_urls = [link.replace("getImage", "getFImage")
413
                          for link in image_urls]
414

415 1
        return image_urls
416

417 1
    def extract_urls(self, html_in):
418
        """
419
        Helper function that uses regexps to extract the image urls from the
420
        given HTML.
421

422
        Parameters
423
        ----------
424
        html_in : str
425
            source from which the urls are to be extracted.
426

427
        Returns
428
        -------
429
        links : list
430
            The list of URLS extracted from the input.
431
        """
432
        # Parse html input for links
433 1
        ahref = re.compile(r'href="([a-zA-Z0-9_\.&\?=%/:-]+)"')
434 1
        links = ahref.findall(html_in)
435 1
        return links
436

437 1
    def query_region(self, coordinates, radius=1 * u.arcmin,
438
                     programme_id=None, database=None,
439
                     verbose=False, get_query_payload=False, system='J2000',
440
                     attributes=['default'], constraints=''):
441
        """
442
        Used to query a region around a known identifier or given
443
        coordinates from the catalog.
444

445
        Parameters
446
        ----------
447
        coordinates : str or `astropy.coordinates` object
448
            The target around which to search. It may be specified as a string
449
            in which case it is resolved using online services or as the
450
            appropriate `astropy.coordinates` object. ICRS coordinates may also
451
            be entered as strings as specified in the `astropy.coordinates`
452
            module.
453
        radius : str or `~astropy.units.Quantity` object, optional
454
            The string must be parsable by `~astropy.coordinates.Angle`. The
455
            appropriate `~astropy.units.Quantity` object from
456
            `astropy.units` may also be used. When missing defaults to 1
457
            arcmin. Cannot exceed 90 arcmin.
458
        programme_id : str
459
            The survey or programme in which to search for. See
460
            `list_catalogs`.
461
        database : str
462
            The WFAU database to use.
463
        verbose : bool, optional.
464
            When set to `True` displays warnings if the returned VOTable does
465
            not conform to the standard. Defaults to `False`.
466
        get_query_payload : bool, optional
467
            If `True` then returns the dictionary sent as the HTTP request.
468
            Defaults to `False`.
469
        system : 'J2000' or 'Galactic'
470
            The system in which to perform the query. Can affect the output
471
            data columns.
472
        attributes : list, optional.
473
            Attributes to select from the table.  See, e.g.,
474
            http://horus.roe.ac.uk/vsa/crossID_notes.html
475
        constraints : str, optional
476
            SQL constraints to the search. Default is empty (no constrains
477
            applied).
478

479
        Returns
480
        -------
481
        result : `~astropy.table.Table`
482
            Query result table.
483
        """
484

485 1
        if database is None:
486 1
            database = self.database
487

488 1
        if programme_id is None:
489 0
            if self.programme_id != 'all':
490 0
                programme_id = self.programme_id
491
            else:
492 0
                raise ValueError("Must specify a programme_id for region queries")
493

494 1
        response = self.query_region_async(coordinates, radius=radius,
495
                                           programme_id=programme_id,
496
                                           database=database,
497
                                           get_query_payload=get_query_payload,
498
                                           system=system, attributes=attributes,
499
                                           constraints=constraints)
500 1
        if get_query_payload:
501 0
            return response
502

503 1
        result = self._parse_result(response, verbose=verbose)
504 1
        return result
505

506 1
    def query_region_async(self, coordinates, radius=1 * u.arcmin,
507
                           programme_id=None,
508
                           database=None, get_query_payload=False,
509
                           system='J2000', attributes=['default'],
510
                           constraints=''):
511
        """
512
        Serves the same purpose as `query_region`. But
513
        returns the raw HTTP response rather than the parsed result.
514

515
        Parameters
516
        ----------
517
        coordinates : str or `astropy.coordinates` object
518
            The target around which to search. It may be specified as a
519
            string in which case it is resolved using online services or as
520
            the appropriate `astropy.coordinates` object. ICRS coordinates
521
            may also be entered as strings as specified in the
522
            `astropy.coordinates` module.
523
        radius : str or `~astropy.units.Quantity` object, optional
524
            The string must be parsable by `~astropy.coordinates.Angle`. The
525
            appropriate `~astropy.units.Quantity` object from
526
            `astropy.units` may also be used. When missing defaults to 1
527
            arcmin. Cannot exceed 90 arcmin.
528
        programme_id : str
529
            The survey or programme in which to search for. See
530
            `list_catalogs`.
531
        database : str
532
            The WFAU database to use.
533
        get_query_payload : bool, optional
534
            If `True` then returns the dictionary sent as the HTTP request.
535
            Defaults to `False`.
536
        attributes : list, optional.
537
            Attributes to select from the table.  See, e.g.,
538
            http://horus.roe.ac.uk/vsa/crossID_notes.html
539
        constraints : str, optional
540
            SQL constraints to the search. Default is empty (no constrains
541
            applied).
542

543
        Returns
544
        -------
545
        response : `requests.Response`
546
            The HTTP response returned from the service.
547
        """
548

549 1
        if database is None:
550 1
            database = self.database
551

552 1
        if programme_id is None:
553 0
            if self.programme_id != 'all':
554 0
                programme_id = self.programme_id
555
            else:
556 0
                raise ValueError("Must specify a programme_id for region queries")
557

558 1
        request_payload = self._args_to_payload(coordinates,
559
                                                programme_id=programme_id,
560
                                                database=database,
561
                                                system=system,
562
                                                query_type='catalog')
563 1
        request_payload['radius'] = _parse_dimension(radius)
564 1
        request_payload['from'] = 'source'
565 1
        request_payload['formaction'] = 'region'
566 1
        request_payload['xSize'] = ''
567 1
        request_payload['ySize'] = ''
568 1
        request_payload['boxAlignment'] = 'RADec'
569 1
        request_payload['emailAddress'] = ''
570 1
        request_payload['format'] = 'VOT'
571 1
        request_payload['compress'] = 'NONE'
572 1
        request_payload['rows'] = 1
573 1
        request_payload['select'] = ','.join(attributes)
574 1
        request_payload['where'] = constraints
575

576
        # for some reason, this is required on the VISTA website
577 1
        if self.archive is not None:
578 0
            request_payload['archive'] = self.archive
579

580 1
        if get_query_payload:
581 1
            return request_payload
582

583 1
        response = self._wfau_send_request(self.REGION_URL, request_payload)
584 1
        response = self._check_page(response.url, "query finished")
585

586 1
        return response
587

588 1
    def _parse_result(self, response, verbose=False):
589
        """
590
        Parses the raw HTTP response and returns it as a
591
        `~astropy.table.Table`.
592

593
        Parameters
594
        ----------
595
        response : `requests.Response`
596
            The HTTP response object
597
        verbose : bool, optional
598
            Defaults to `False`. If `True` it displays warnings whenever the
599
            VOtable returned from the service doesn't conform to the standard.
600

601
        Returns
602
        -------
603
        table : `~astropy.table.Table`
604
        """
605 1
        table_links = self.extract_urls(response.text)
606
        # keep only one link that is not a webstart
607 1
        if len(table_links) == 0:
608 0
            raise Exception("No VOTable found on returned webpage!")
609 1
        table_link = [link for link in table_links if "8080" not in link][0]
610 1
        with commons.get_readable_fileobj(table_link) as f:
611 1
            content = f.read()
612

613 1
        if not verbose:
614 1
            commons.suppress_vo_warnings()
615

616 1
        try:
617 1
            io_obj = BytesIO(content.encode('utf-8'))
618 1
            parsed_table = votable.parse(io_obj, pedantic=False)
619 1
            first_table = parsed_table.get_first_table()
620 1
            table = first_table.to_table()
621 1
            if len(table) == 0:
622 0
                warnings.warn("Query returned no results, so the table will "
623
                              "be empty", NoResultsWarning)
624 1
            return table
625 0
        except Exception as ex:
626 0
            self.response = content
627 0
            self.table_parse_error = ex
628 0
            raise
629 0
            raise TableParseError("Failed to parse WFAU votable! The raw "
630
                                  "response can be found in self.response, "
631
                                  "and the error in self.table_parse_error.  "
632
                                  "Exception: " + str(self.table_parse_error))
633

634 1
    def list_catalogs(self, style='short'):
635
        """
636
        Returns a list of available catalogs in WFAU.
637
        These can be used as ``programme_id`` in queries.
638

639
        Parameters
640
        ----------
641
        style : str, optional
642
            Must be one of ``'short'``, ``'long'``. Defaults to ``'short'``.
643
            Determines whether to print long names or abbreviations for
644
            catalogs.
645

646
        Returns
647
        -------
648
        list : list containing catalog name strings in long or short style.
649
        """
650 0
        if style == 'short':
651 0
            return list(self.programmes_short.keys())
652 0
        elif style == 'long':
653 0
            return list(self.programmes_long.keys())
654
        else:
655 0
            warnings.warn("Style must be one of 'long', 'short'.\n"
656
                          "Returning catalog list in short format.\n")
657 0
            return list(self.programmes_short.keys())
658

659 1
    def _get_databases(self):
660 0
        if self.logged_in():
661 0
            response = self.session.get(url="/".join([self.BASE_URL,
662
                                                      self.IMAGE_FORM]))
663
        else:
664 0
            response = requests.get(url="/".join([self.BASE_URL,
665
                                                  self.IMAGE_FORM]))
666

667 0
        root = BeautifulSoup(response.content, features='html5lib')
668 0
        databases = [x.attrs['value'] for x in
669
                     root.find('select').findAll('option')]
670 0
        return databases
671

672 1
    def list_databases(self):
673
        """
674
        List the databases available from the WFAU archive.
675
        """
676 0
        self.databases = set(self.all_databases + tuple(self._get_databases()))
677 0
        return self.databases
678

679 1
    def _wfau_send_request(self, url, request_payload):
680
        """
681
        Helper function that sends the query request via a session or simple
682
        HTTP GET request.
683

684
        Parameters
685
        ----------
686
        url : str
687
            The url to send the request to.
688
        request_payload : dict
689
            The dict of parameters for the GET request
690

691
        Returns
692
        -------
693
        response : `requests.Response` object
694
            The response for the HTTP GET request
695
        """
696 1
        if hasattr(self, 'session') and self.logged_in():
697 0
            response = self.session.get(url, params=request_payload,
698
                                        timeout=self.TIMEOUT)
699
        else:
700 1
            response = self._request("GET", url=url, params=request_payload,
701
                                     timeout=self.TIMEOUT)
702 1
        return response
703

704 1
    def _check_page(self, url, keyword, wait_time=1, max_attempts=30):
705 1
        page_loaded = False
706 1
        while not page_loaded and max_attempts > 0:
707 1
            if self.logged_in():
708 0
                response = self.session.get(url)
709
            else:
710 1
                response = requests.get(url=url)
711 1
            self.response = response
712 1
            content = response.text
713 1
            if re.search("error", content, re.IGNORECASE):
714 1
                raise InvalidQueryError(
715
                    "Service returned with an error!  "
716
                    "Check self.response for more information.")
717 1
            elif re.search(keyword, content, re.IGNORECASE):
718 1
                page_loaded = True
719 1
            max_attempts -= 1
720
            # wait for wait_time seconds before checking again
721 1
            time.sleep(wait_time)
722 1
        if page_loaded is False:
723 0
            raise TimeoutError("Page did not load.")
724 1
        return response
725

726 1
    def query_cross_id_async(self, coordinates, radius=1*u.arcsec,
727
                             programme_id=None, database=None, table="source",
728
                             constraints="", attributes='default',
729
                             pairing='all', system='J2000',
730
                             get_query_payload=False,
731
                             ):
732
        """
733
        Query the crossID server
734

735
        Parameters
736
        ----------
737
        coordinates : astropy.SkyCoord
738
            An array of one or more astropy SkyCoord objects specifying the
739
            objects to crossmatch against.
740
        radius : str or `~astropy.units.Quantity` object, optional
741
            The string must be parsable by `~astropy.coordinates.Angle`. The
742
            appropriate `~astropy.units.Quantity` object from
743
            `astropy.units` may also be used. When missing defaults to 1
744
            arcsec.
745
        programme_id : str
746
            The survey or programme in which to search for. See
747
            `list_catalogs`.
748
        database : str
749
            The WFAU database to use.
750
        table : str
751
            The table ID, one of: "source", "detection", "synopticSource"
752
        constraints : str
753
            SQL constraints.  If 'source' is selected, this will be expanded
754
            automatically
755
        attributes : str
756
            Additional attributes to select from the table.  See, e.g.,
757
            http://horus.roe.ac.uk/vsa/crossID_notes.html
758
        system : 'J2000' or 'Galactic'
759
            The system in which to perform the query. Can affect the output
760
            data columns.
761
        get_query_payload : bool, optional
762
            If `True` then returns the dictionary sent as the HTTP request.
763
            Defaults to `False`.
764
        """
765

766 0
        if table == "source":
767 0
            constraints += "(priOrSec<=0 OR priOrSec=frameSetID)"
768 0
        if database is None:
769 0
            database = self.database
770

771 0
        if programme_id is None:
772 0
            if self.programme_id != 'all':
773 0
                programme_id = self.programme_id
774
            else:
775 0
                raise ValueError("Must specify a programme_id")
776

777 0
        request_payload = self._args_to_payload(coordinates,
778
                                                programme_id=programme_id,
779
                                                database=database,
780
                                                system=system,
781
                                                query_type='catalog')
782 0
        request_payload['radius'] = _parse_dimension(radius)
783 0
        request_payload['from'] = 'source'
784 0
        request_payload['formaction'] = 'region'
785 0
        request_payload['xSize'] = ''
786 0
        request_payload['ySize'] = ''
787 0
        request_payload['boxAlignment'] = 'RADec'
788 0
        request_payload['emailAddress'] = ''
789 0
        request_payload['format'] = 'VOT'
790 0
        request_payload['compress'] = 'NONE'
791 0
        request_payload['rows'] = 1
792 0
        request_payload['select'] = 'default'
793 0
        request_payload['where'] = ''
794 0
        request_payload['disp'] = ''
795 0
        request_payload['baseTable'] = table
796 0
        request_payload['whereClause'] = constraints
797 0
        request_payload['qType'] = 'form'
798 0
        request_payload['selectList'] = attributes
799 0
        request_payload['uploadFile'] = 'file.txt'
800 0
        if pairing not in ('nearest', 'all'):
801 0
            raise ValueError("pairing must be one of 'nearest' or 'all'")
802 0
        request_payload['nearest'] = 0 if pairing == 'nearest' else 1
803

804
        # for some reason, this is required on the VISTA website
805 0
        if self.archive is not None:
806 0
            request_payload['archive'] = self.archive
807

808 0
        if get_query_payload:
809 0
            return request_payload
810

811 0
        fh = StringIO()
812 0
        assert len(coordinates) > 0
813 0
        for crd in coordinates:
814 0
            fh.write("{0} {1}\n".format(crd.ra.deg, crd.dec.deg))
815 0
        fh.seek(0)
816

817 0
        if hasattr(self, 'session') and self.logged_in():
818 0
            response = self.session.post(self.CROSSID_URL,
819
                                         params=request_payload,
820
                                         files={'file.txt': fh},
821
                                         timeout=self.TIMEOUT)
822
        else:
823 0
            response = self._request("POST", url=self.CROSSID_URL,
824
                                     params=request_payload,
825
                                     files={'file.txt': fh},
826
                                     timeout=self.TIMEOUT)
827

828 0
        raise NotImplementedError("It appears we haven't implemented the file "
829
                                  "upload correctly.  Help is needed.")
830

831
        # response = self._check_page(response.url, "query finished")
832

833 0
        return response
834

835 1
    def query_cross_id(self, *args, **kwargs):
836
        """
837
        See `query_cross_id_async`
838
        """
839 0
        get_query_payload = kwargs.get('get_query_payload', False)
840 0
        verbose = kwargs.get('verbose', False)
841

842 0
        response = self.query_cross_id_async(*args, **kwargs)
843

844 0
        if get_query_payload:
845 0
            return response
846

847 0
        result = self._parse_result(response, verbose=verbose)
848 0
        return result
849

850

851 1
def clean_catalog(wfau_catalog, clean_band='K_1', badclass=-9999,
852
                  maxerrbits=41, minerrbits=0, maxpperrbits=60):
853
    """
854
    Attempt to remove 'bad' entries in a catalog.
855

856
    Parameters
857
    ----------
858
    wfau_catalog : `~astropy.io.fits.BinTableHDU`
859
        A FITS binary table instance from the WFAU survey.
860
    clean_band : ``'K_1'``, ``'K_2'``, ``'J'``, ``'H'``
861
        The band to use for bad photometry flagging.
862
    badclass : int
863
        Class to exclude.
864
    minerrbits : int
865
    maxerrbits : int
866
        Inside this range is the accepted number of error bits.
867
    maxpperrbits : int
868
        Exclude this type of error bit.
869

870
    Examples
871
    --------
872
    """
873

874 0
    band = clean_band
875 0
    mask = ((wfau_catalog[band + 'ERRBITS'] <= maxerrbits) *
876
            (wfau_catalog[band + 'ERRBITS'] >= minerrbits) *
877
            ((wfau_catalog['PRIORSEC'] == wfau_catalog['FRAMESETID']) +
878
             (wfau_catalog['PRIORSEC'] == 0)) *
879
            (wfau_catalog[band + 'PPERRBITS'] < maxpperrbits)
880
            )
881 0
    if band + 'CLASS' in wfau_catalog.colnames:
882 0
        mask *= (wfau_catalog[band + 'CLASS'] != badclass)
883 0
    elif 'mergedClass' in wfau_catalog.colnames:
884 0
        mask *= (wfau_catalog['mergedClass'] != badclass)
885

886 0
    return wfau_catalog.data[mask]
887

888

889 1
def _parse_dimension(dim):
890
    """
891
    Parses the radius and returns it in the format expected by WFAU.
892

893
    Parameters
894
    ----------
895
    dim : str, `~astropy.units.Quantity`
896

897
    Returns
898
    -------
899
    dim_in_min : float
900
        The value of the radius in arcminutes.
901
    """
902 1
    if (isinstance(dim, u.Quantity) and
903
            dim.unit in u.deg.find_equivalent_units()):
904 1
        dim_in_min = dim.to(u.arcmin).value
905
    # otherwise must be an Angle or be specified in hours...
906
    else:
907 0
        try:
908 0
            new_dim = coord.Angle(dim).degree
909 0
            dim_in_min = u.Quantity(
910
                value=new_dim, unit=u.deg).to(u.arcmin).value
911 0
        except (u.UnitsError, coord.errors.UnitsError, AttributeError):
912 0
            raise u.UnitsError("Dimension not in proper units")
913 1
    return dim_in_min

Read our documentation on viewing source code .

Loading