astropy / astroquery
1
# Licensed under a 3-clause BSD style license - see LICENSE.rst
2 1
from __future__ import print_function
3 1
import json
4 1
import os
5 1
import tempfile
6 1
import tarfile
7 1
import sys
8

9 1
import six
10 1
from astropy.io import fits
11 1
from astropy import log
12 1
import astropy.units
13 1
import astropy.io.votable as votable
14 1
from requests import HTTPError
15

16 1
from ..query import BaseQuery
17 1
from ..utils import commons
18 1
from ..utils import async_to_sync
19 1
from . import conf
20 1
from ..exceptions import TableParseError
21 1
from .. import version
22 1
from astropy.coordinates.name_resolve import sesame_database
23

24

25 1
@async_to_sync
26 1
class ESASkyClass(BaseQuery):
27

28 1
    URLbase = conf.urlBase
29 1
    TIMEOUT = conf.timeout
30 1
    DEFAULT_ROW_LIMIT = conf.row_limit
31

32 1
    __FITS_STRING = ".fits"
33 1
    __FTZ_STRING = ".FTZ"
34 1
    __TAR_STRING = ".tar"
35 1
    __ALL_STRING = "all"
36 1
    __CATALOGS_STRING = "catalogs"
37 1
    __OBSERVATIONS_STRING = "observations"
38 1
    __MISSION_STRING = "mission"
39 1
    __TAP_TABLE_STRING = "tapTable"
40 1
    __TAP_NAME_STRING = "tapName"
41 1
    __LABEL_STRING = "label"
42 1
    __METADATA_STRING = "metadata"
43 1
    __PRODUCT_URL_STRING = "product_url"
44 1
    __SOURCE_LIMIT_STRING = "sourceLimit"
45 1
    __POLYGON_NAME_STRING = "polygonNameTapColumn"
46 1
    __POLYGON_RA_STRING = "polygonRaTapColumn"
47 1
    __POLYGON_DEC_STRING = "polygonDecTapColumn"
48 1
    __POS_TAP_STRING = "posTapColumn"
49 1
    __ORDER_BY_STRING = "orderBy"
50 1
    __IS_SURVEY_MISSION_STRING = "isSurveyMission"
51 1
    __ZERO_ARCMIN_STRING = "0 arcmin"
52 1
    __MIN_RADIUS_CATALOG_STRING = "5 arcsec"
53

54 1
    __HERSCHEL_STRING = 'herschel'
55 1
    __HST_STRING = 'hst'
56 1
    __INTEGRAL_STRING = 'integral'
57 1
    __AKARI_STRING = 'akari'
58

59 1
    __HERSCHEL_FILTERS = {
60
        'psw': '250',
61
        'pmw': '350',
62
        'plw': '500',
63
        'mapb_blue': '70',
64
        'mapb_green': '100',
65
        'mapr_': '160'}
66

67 1
    _MAPS_DOWNLOAD_DIR = "Maps"
68 1
    _isTest = ""
69

70 1
    def list_maps(self):
71
        """
72
        Get a list of the mission names of the available observations in ESASky
73
        """
74 0
        return self._json_object_field_to_list(
75
            self._get_observation_json(), self.__MISSION_STRING)
76

77 1
    def list_catalogs(self):
78
        """
79
        Get a list of the mission names of the available catalogs in ESASky
80
        """
81 1
        return self._json_object_field_to_list(
82
            self._get_catalogs_json(), self.__MISSION_STRING)
83

84 1
    def query_object_maps(self, position, missions=__ALL_STRING,
85
                          get_query_payload=False, cache=True):
86
        """
87
        This method queries a chosen object or coordinate for all available maps
88
        which have observation data on the chosen position. It returns a
89
        TableList with all the found maps metadata for the chosen missions
90
        and object.
91

92
        Parameters
93
        ----------
94
        position : str or `astropy.coordinates` object
95
            Can either be a string of the location, eg 'M51', or the coordinates
96
            of the object.
97
        missions : string or list, optional
98
            Can be either a specific mission or a list of missions (all mission
99
            names are found in list_missions()) or 'all' to search in all
100
            missions. Defaults to 'all'.
101
        get_query_payload : bool, optional
102
            When set to True the method returns the HTTP request parameters.
103
            Defaults to False.
104
        cache : bool, optional
105
            When set to True the method will use a cache located at
106
            .astropy/astroquery/cache. Defaults to True.
107

108
        Returns
109
        -------
110
        table_list : `~astroquery.utils.TableList`
111
            Each mission returns a `~astropy.table.Table` with the metadata
112
            and observations available for the chosen missions and object.
113
            It is structured in a TableList like this:
114
            TableList with 8 tables:
115
            '0:HERSCHEL' with 8 column(s) and 25 row(s)
116
            '1:HST' with 8 column(s) and 735 row(s)
117

118
        Examples
119
        --------
120
        query_object_maps("m101", "all")
121

122
        query_object_maps("265.05, 69.0", "Herschel")
123
        query_object_maps("265.05, 69.0", ["Herschel", "HST"])
124
        """
125 0
        return self.query_region_maps(position=position,
126
                                      radius=self.__ZERO_ARCMIN_STRING,
127
                                      missions=missions,
128
                                      get_query_payload=get_query_payload,
129
                                      cache=cache)
130

131 1
    def query_object_catalogs(self, position, catalogs=__ALL_STRING,
132
                              row_limit=DEFAULT_ROW_LIMIT,
133
                              get_query_payload=False, cache=True):
134
        """
135
        This method queries a chosen object or coordinate for all available
136
        catalogs and returns a TableList with all the found catalogs metadata
137
        for the chosen missions and object. To account for errors in telescope
138
        position, the method will look for any sources within a radius of
139
        5 arcsec of the chosen position.
140

141
        Parameters
142
        ----------
143
        position : str or `astropy.coordinates` object
144
            Can either be a string of the location, eg 'M51', or the coordinates
145
            of the object.
146
        catalogs : string or list, optional
147
            Can be either a specific catalog or a list of catalogs (all catalog
148
            names are found in list_catalogs()) or 'all' to search in all
149
            catalogs. Defaults to 'all'.
150
        row_limit : int, optional
151
            Determines how many rows that will be fetched from the database
152
            for each mission. Can be -1 to select maximum (currently 100 000).
153
            Defaults to 10000.
154
        get_query_payload : bool, optional
155
            When set to True the method returns the HTTP request parameters.
156
            Defaults to False.
157
        cache : bool, optional
158
            When set to True the method will use a cache located at
159
            .astropy/astroquery/cache. Defaults to True.
160
        Returns
161
        -------
162
        table_list : `~astroquery.utils.TableList`
163
            Each mission returns a `~astropy.table.Table` with the metadata
164
            of the catalogs available for the chosen mission and object.
165
            It is structured in a TableList like this:
166
            TableList with 8 tables:
167
            '0:Gaia DR1 TGA' with 8 column(s) and 25 row(s)
168
            '1:HSC' with 8 column(s) and 75 row(s)
169

170
        Examples
171
        --------
172
        query_object_catalogs("m101", "all")
173

174
        query_object_catalogs("265.05, 69.0", "Gaia DR1 TGA")
175
        query_object_catalogs("265.05, 69.0", ["Gaia DR1 TGA", "HSC"])
176
        """
177 0
        return self.query_region_catalogs(position=position,
178
                                          radius=self.__ZERO_ARCMIN_STRING,
179
                                          catalogs=catalogs,
180
                                          row_limit=row_limit,
181
                                          get_query_payload=get_query_payload,
182
                                          cache=cache)
183

184 1
    def query_region_maps(self, position, radius, missions=__ALL_STRING,
185
                          get_query_payload=False, cache=True):
186
        """
187
        This method queries a chosen region for all available maps and returns a
188
        TableList with all the found maps metadata for the chosen missions and
189
        region.
190

191
        Parameters
192
        ----------
193
        position : str or `astropy.coordinates` object
194
            Can either be a string of the location, eg 'M51', or the coordinates
195
            of the object.
196
        radius : str or `~astropy.units.Quantity`
197
            The radius of a region.
198
        missions : string or list, optional
199
            Can be either a specific mission or a list of missions (all mission
200
            names are found in list_missions()) or 'all' to search in all
201
            missions. Defaults to 'all'.
202
        get_query_payload : bool, optional
203
            When set to True the method returns the HTTP request parameters.
204
            Defaults to False.
205
        cache : bool, optional
206
            When set to True the method will use a cache located at
207
            .astropy/astroquery/cache. Defaults to True.
208

209
        Returns
210
        -------
211
        table_list : `~astroquery.utils.TableList`
212
            Each mission returns a `~astropy.table.Table` with the metadata
213
            and observations available for the chosen missions and region.
214
            It is structured in a TableList like this:
215
            TableList with 8 tables:
216
            '0:HERSCHEL' with 8 column(s) and 25 row(s)
217
            '1:HST' with 8 column(s) and 735 row(s)
218

219
        Examples
220
        --------
221
        query_region_maps("m101", "14'", "all")
222

223
        import astropy.units as u
224
        query_region_maps("265.05, 69.0", 14*u.arcmin, "Herschel")
225
        query_region_maps("265.05, 69.0", ["Herschel", "HST"])
226
        """
227 1
        sanitized_position = self._sanitize_input_position(position)
228 1
        sanitized_radius = self._sanitize_input_radius(radius)
229 1
        sanitized_missions = self._sanitize_input_mission(missions)
230

231 0
        query_result = {}
232

233 0
        sesame_database.set('simbad')
234 0
        coordinates = commons.parse_coordinates(sanitized_position)
235

236 0
        self._store_query_result_maps(query_result, sanitized_missions,
237
                                      coordinates, sanitized_radius,
238
                                      get_query_payload, cache)
239

240 0
        if (get_query_payload):
241 0
            return query_result
242

243 0
        return commons.TableList(query_result)
244

245 1
    def query_region_catalogs(self, position, radius, catalogs=__ALL_STRING,
246
                              row_limit=DEFAULT_ROW_LIMIT,
247
                              get_query_payload=False, cache=True):
248
        """
249
        This method queries a chosen region for all available catalogs and
250
        returns a TableList with all the found catalogs metadata for the chosen
251
        missions and region.
252

253
        Parameters
254
        ----------
255
        position : str or `astropy.coordinates` object
256
            Can either be a string of the location, eg 'M51', or the coordinates
257
            of the object.
258
        radius : str or `~astropy.units.Quantity`
259
            The radius of a region.
260
        catalogs : string or list, optional
261
            Can be either a specific catalog or a list of catalogs (all catalog
262
            names are found in list_catalogs()) or 'all' to search in all
263
            catalogs. Defaults to 'all'.
264
        row_limit : int, optional
265
            Determines how many rows that will be fetched from the database
266
            for each mission. Can be -1 to select maximum (currently 100 000).
267
            Defaults to 10000.
268
        get_query_payload : bool, optional
269
            When set to True the method returns the HTTP request parameters.
270
            Defaults to False.
271
        cache : bool, optional
272
            When set to True the method will use a cache located at
273
            .astropy/astroquery/cache. Defaults to True.
274

275
        Returns
276
        -------
277
        table_list : `~astroquery.utils.TableList`
278
            Each mission returns a `~astropy.table.Table` with the metadata of
279
            the catalogs available for the chosen mission and region.
280
            It is structured in a TableList like this:
281
            TableList with 8 tables:
282
            '0:Gaia DR1 TGA' with 8 column(s) and 25 row(s)
283
            '1:HSC' with 8 column(s) and 75 row(s)
284

285
        Examples
286
        --------
287
        query_region_catalogs("m101", "14'", "all")
288

289
        import astropy.units as u
290
        query_region_catalogs("265.05, 69.0", 14*u.arcmin, "Gaia DR1 TGA")
291
        query_region_catalogs("265.05, 69.0", 14*u.arcmin, ["Gaia DR1 TGA", "HSC"])
292
        """
293 0
        sanitized_position = self._sanitize_input_position(position)
294 0
        sanitized_radius = self._sanitize_input_radius(radius)
295 0
        sanitized_catalogs = self._sanitize_input_catalogs(catalogs)
296 0
        sanitized_row_limit = self._sanitize_input_row_limit(row_limit)
297

298 0
        sesame_database.set('simbad')
299 0
        coordinates = commons.parse_coordinates(sanitized_position)
300

301 0
        query_result = {}
302

303 0
        self._store_query_result_catalogs(query_result, sanitized_catalogs,
304
                                          coordinates, sanitized_radius,
305
                                          sanitized_row_limit,
306
                                          get_query_payload, cache)
307

308 0
        if (get_query_payload):
309 0
            return query_result
310

311 0
        return commons.TableList(query_result)
312

313 1
    def get_maps(self, query_table_list, missions=__ALL_STRING,
314
                 download_dir=_MAPS_DOWNLOAD_DIR, cache=True):
315
        """
316
        This method takes the dictionary of missions and metadata as returned by
317
        query_region_maps and downloads all maps to the selected folder.
318
        The method returns a dictionary which is divided by mission.
319
        All mission except Herschel returns a list of HDULists.
320
        For Herschel each item in the list is a dictionary where the used
321
        filter is the key and the HDUList is the value.
322

323
        Parameters
324
        ----------
325
        query_table_list : `~astroquery.utils.TableList` or dict or list of (name, `~astropy.table.Table`) pairs
326
            A TableList or dict or list of name and Table pairs with all the
327
            missions wanted and their respective metadata. Usually the
328
            return value of query_region_maps.
329
        missions : string or list, optional
330
            Can be either a specific mission or a list of missions (all mission
331
            names are found in list_missions()) or 'all' to search in all
332
            missions. Defaults to 'all'.
333
        download_dir : string, optional
334
            The folder where all downloaded maps should be stored.
335
            Defaults to a folder called 'Maps' in the current working directory.
336
        cache : bool, optional
337
            When set to True the method will use a cache located at
338
            .astropy/astroquery/cache. Defaults to True.
339

340
        Returns
341
        -------
342
        maps : `dict`
343
            All mission except Herschel returns a list of HDULists.
344
            For Herschel each item in the list is a dictionary where the used
345
            filter is the key and the HDUList is the value.
346
            It is structured in a dictionary like this:
347
            dict: {
348
            'HERSCHEL': [{'70': [HDUList], '160': [HDUList]}, {'70': [HDUList], '160': [HDUList]}, ...],
349
            'HST':[[HDUList], [HDUList], [HDUList], [HDUList], [HDUList], ...],
350
            'XMM-EPIC' : [[HDUList], [HDUList], [HDUList], [HDUList], ...]
351
            ...
352
            }
353

354
        Examples
355
        --------
356
        get_maps(query_region_catalogs("m101", "14'", "all"))
357

358
        """
359 0
        sanitized_query_table_list = self._sanitize_input_table_list(query_table_list)
360 0
        sanitized_missions = [m.lower() for m in self._sanitize_input_mission(missions)]
361

362 0
        maps = dict()
363

364 0
        for query_mission in sanitized_query_table_list.keys():
365

366 0
            if (query_mission.lower() in sanitized_missions):
367
                # INTEGRAL & AKARI does not have a product url yet.
368 0
                if (query_mission.lower() == self.__INTEGRAL_STRING
369
                    or query_mission.lower() == self.__AKARI_STRING):
370 0
                    log.info(query_mission + " does not yet support downloading of "
371
                            "fits files")
372 0
                    continue
373 0
                maps[query_mission] = (
374
                    self._get_maps_for_mission(
375
                        sanitized_query_table_list[query_mission],
376
                        query_mission,
377
                        download_dir,
378
                        cache))
379

380 0
        if all([maps[mission].count(None) == len(maps[mission])
381
                for mission in maps]):
382 0
            log.info("No maps got downloaded, check errors above.")
383

384 0
        elif (len(sanitized_query_table_list) > 0):
385 0
            log.info("Maps available at {}.".format(os.path.abspath(download_dir)))
386
        else:
387 0
            log.info("No maps found.")
388 0
        return maps
389

390 1
    def get_images(self, position, radius=__ZERO_ARCMIN_STRING, missions=__ALL_STRING,
391
                   download_dir=_MAPS_DOWNLOAD_DIR, cache=True):
392
        """
393
        This method gets the fits files available for the selected position and
394
        mission and downloads all maps to the the selected folder.
395
        The method returns a dictionary which is divided by mission.
396
        All mission except Herschel returns a list of HDULists.
397
        For Herschel each item in the list is a dictionary where the used
398
        filter is the key and the HDUList is the value.
399

400
        Parameters
401
        ----------
402
        position : str or `astropy.coordinates` object
403
            Can either be a string of the location, eg 'M51', or the coordinates
404
            of the object.
405
        radius : str or `~astropy.units.Quantity`, optional
406
            The radius of a region. Defaults to 0.
407
        missions : string or list, optional
408
            Can be either a specific mission or a list of missions (all mission
409
            names are found in list_missions()) or 'all' to search in all
410
            missions. Defaults to 'all'.
411
        download_dir : string, optional
412
            The folder where all downloaded maps should be stored.
413
            Defaults to a folder called 'Maps' in the current working directory.
414
        cache : bool, optional
415
            When set to True the method will use a cache located at
416
            .astropy/astroquery/cache. Defaults to True.
417

418
        Returns
419
        -------
420
        maps : `dict`
421
            All mission except Herschel returns a list of HDULists.
422
            For Herschel each item in the list is a dictionary where the used
423
            filter is the key and the HDUList is the value.
424
            It is structured in a dictionary like this:
425
            dict: {
426
            'HERSCHEL': [{'70': [HDUList], '160': [HDUList]}, {'70': [HDUList], '160': [HDUList]}, ...],
427
            'HST':[[HDUList], [HDUList], [HDUList], [HDUList], [HDUList], ...],
428
            'XMM-EPIC' : [[HDUList], [HDUList], [HDUList], [HDUList], ...]
429
            ...
430
            }
431

432
        Examples
433
        --------
434
        get_images("m101", "14'", "all")
435

436
        """
437 0
        sanitized_position = self._sanitize_input_position(position)
438 0
        sanitized_radius = self._sanitize_input_radius(radius)
439 0
        sanitized_missions = self._sanitize_input_mission(missions)
440

441 0
        maps = dict()
442

443 0
        map_query_result = self.query_region_maps(sanitized_position,
444
                                                  sanitized_radius,
445
                                                  sanitized_missions,
446
                                                  get_query_payload=False,
447
                                                  cache=cache)
448

449 0
        for query_mission in map_query_result.keys():
450
            # INTEGRAL & AKARI does not have a product url yet.
451 0
            if (query_mission.lower() == self.__INTEGRAL_STRING
452
                or query_mission.lower() == self.__AKARI_STRING):
453 0
                log.info(query_mission + " does not yet support downloading of "
454
                        "fits files")
455 0
                continue
456 0
            maps[query_mission] = (
457
                self._get_maps_for_mission(
458
                    map_query_result[query_mission],
459
                    query_mission,
460
                    download_dir,
461
                    cache))
462

463 0
        if all([maps[mission].count(None) == len(maps[mission])
464
                for mission in maps]):
465 0
            log.info("No maps got downloaded, check errors above.")
466 0
        elif (len(map_query_result) > 0):
467 0
            log.info("Maps available at {}".format(os.path.abspath(download_dir)))
468
        else:
469 0
            log.info("No maps found.")
470 0
        return maps
471

472 1
    def _sanitize_input_position(self, position):
473 1
        if (isinstance(position, str) or isinstance(position,
474
                                                    commons.CoordClasses)):
475 1
            return position
476
        else:
477 1
            raise ValueError("Position must be either a string or "
478
                             "astropy.coordinates")
479

480 1
    def _sanitize_input_radius(self, radius):
481 1
        if (isinstance(radius, str) or isinstance(radius,
482
                                                  astropy.units.Quantity)):
483 1
            return radius
484
        else:
485 1
            raise ValueError("Radius must be either a string or "
486
                             "astropy.units.Quantity")
487

488 1
    def _sanitize_input_mission(self, missions):
489 1
        if isinstance(missions, list):
490 0
            return missions
491 1
        if isinstance(missions, str):
492 0
            if (missions.lower() == self.__ALL_STRING):
493 0
                return self.list_maps()
494
            else:
495 0
                return [missions]
496 1
        raise ValueError("Mission must be either a string or a list of "
497
                         "missions")
498

499 1
    def _sanitize_input_catalogs(self, catalogs):
500 0
        if isinstance(catalogs, list):
501 0
            return catalogs
502 0
        if isinstance(catalogs, str):
503 0
            if (catalogs.lower() == self.__ALL_STRING):
504 0
                return self.list_catalogs()
505
            else:
506 0
                return [catalogs]
507 0
        raise ValueError("Catalog must be either a string or a list of "
508
                         "catalogs")
509

510 1
    def _sanitize_input_table_list(self, table_list):
511 0
        if isinstance(table_list, commons.TableList):
512 0
            return table_list
513

514 0
        try:
515 0
            return commons.TableList(table_list)
516 0
        except ValueError:
517 0
            raise ValueError(
518
                "query_table_list must be an astroquery.utils.TableList "
519
                "or be able to be converted to it.")
520

521 1
    def _sanitize_input_row_limit(self, row_limit):
522 0
        if isinstance(row_limit, int):
523 0
            return row_limit
524 0
        raise ValueError("Row_limit must be an integer")
525

526 1
    def _get_maps_for_mission(self, maps_table, mission, download_dir, cache):
527 0
        maps = []
528

529 0
        if (len(maps_table[self.__PRODUCT_URL_STRING]) > 0):
530 0
            mission_directory = self._create_mission_directory(mission,
531
                                                               download_dir)
532 0
            log.info("Starting download of {} data. ({} files)".format(
533
                mission, len(maps_table[self.__PRODUCT_URL_STRING])))
534 0
            for index in range(len(maps_table)):
535 0
                product_url = maps_table[self.__PRODUCT_URL_STRING][index]
536 0
                if commons.ASTROPY_LT_4_1:
537 0
                    product_url = product_url.decode('utf-8')
538 0
                if(mission.lower() == self.__HERSCHEL_STRING):
539 0
                    observation_id = maps_table["observation_id"][index]
540 0
                    if commons.ASTROPY_LT_4_1:
541 0
                        observation_id = observation_id.decode('utf-8')
542
                else:
543 0
                    observation_id = maps_table[self._get_tap_observation_id(mission)][index]
544 0
                    if commons.ASTROPY_LT_4_1:
545 0
                        observation_id = observation_id.decode('utf-8')
546 0
                log.info("Downloading Observation ID: {} from {}"
547
                         .format(observation_id, product_url))
548 0
                sys.stdout.flush()
549 0
                directory_path = mission_directory + "/"
550 0
                if (mission.lower() == self.__HERSCHEL_STRING):
551 0
                    try:
552 0
                        maps.append(self._get_herschel_map(
553
                            product_url,
554
                            directory_path,
555
                            cache))
556 0
                    except HTTPError as err:
557 0
                        log.error("Download failed with {}.".format(err))
558 0
                        maps.append(None)
559

560
                else:
561 0
                    response = self._request(
562
                        'GET',
563
                        product_url,
564
                        cache=cache,
565
                        headers=self._get_header())
566

567 0
                    try:
568 0
                        response.raise_for_status()
569

570 0
                        file_name = ""
571 0
                        if (product_url.endswith(self.__FITS_STRING)):
572 0
                            file_name = (directory_path +
573
                                         self._extract_file_name_from_url(product_url))
574
                        else:
575 0
                            file_name = (directory_path +
576
                                         self._extract_file_name_from_response_header(response.headers))
577

578 0
                        fits_data = response.content
579 0
                        with open(file_name, 'wb') as fits_file:
580 0
                            fits_file.write(fits_data)
581 0
                            fits_file.close()
582 0
                            maps.append(fits.open(file_name))
583 0
                    except HTTPError as err:
584 0
                        log.error("Download failed with {}.".format(err))
585 0
                        maps.append(None)
586

587 0
                if None in maps:
588 0
                    log.error("Some downloads were unsuccessful, please check "
589
                              "the warnings for more details")
590

591
                else:
592 0
                    log.info("[Done]")
593

594 0
            log.info("Downloading of {} data complete.".format(mission))
595

596 0
        return maps
597

598 1
    def _get_herschel_map(self, product_url, directory_path, cache):
599 0
        observation = dict()
600 0
        tar_file = tempfile.NamedTemporaryFile(delete=False)
601 0
        response = self._request('GET', product_url, cache=cache,
602
                                 headers=self._get_header())
603

604 0
        response.raise_for_status()
605

606 0
        tar_file.write(response.content)
607 0
        tar_file.close()
608 0
        with tarfile.open(tar_file.name, 'r') as tar:
609 0
            i = 0
610 0
            for member in tar.getmembers():
611 0
                member_name = member.name.lower()
612 0
                if ('hspire' in member_name or 'hpacs' in member_name):
613 0
                    herschel_filter = self._get_herschel_filter_name(member_name)
614 0
                    tar.extract(member, directory_path)
615 0
                    observation[herschel_filter] = fits.open(
616
                        directory_path +
617
                        member.name)
618 0
                    i += 1
619 0
        os.remove(tar_file.name)
620 0
        return observation
621

622 1
    def _get_herschel_filter_name(self, member_name):
623 0
        for herschel_filter in self.__HERSCHEL_FILTERS.keys():
624 0
            if herschel_filter in member_name:
625 0
                return self.__HERSCHEL_FILTERS[herschel_filter]
626

627 1
    def _remove_extra_herschel_directory(self, file_and_directory_name,
628
                                         directory_path):
629 0
        full_directory_path = os.path.abspath(directory_path)
630 0
        file_name = file_and_directory_name[file_and_directory_name.index("/") + 1:]
631

632 0
        os.renames(os.path.join(full_directory_path, file_and_directory_name),
633
                   os.path.join(full_directory_path, file_name))
634 0
        return file_name
635

636 1
    def _create_mission_directory(self, mission, download_dir):
637 0
        if (download_dir == self._MAPS_DOWNLOAD_DIR):
638 0
            mission_directory = self._MAPS_DOWNLOAD_DIR + "/" + mission
639
        else:
640 0
            mission_directory = (download_dir + "/" + self._MAPS_DOWNLOAD_DIR +
641
                                 "/" + mission)
642 0
        if not os.path.exists(mission_directory):
643 0
            os.makedirs(mission_directory)
644 0
        return mission_directory
645

646 1
    def _extract_file_name_from_response_header(self, headers):
647 0
        content_disposition = headers.get('Content-Disposition')
648 0
        filename_string = "filename="
649 0
        start_index = (content_disposition.index(filename_string) +
650
                       len(filename_string))
651 0
        if (content_disposition[start_index] == '\"'):
652 0
            start_index += 1
653

654 0
        if (self.__FITS_STRING in content_disposition[start_index:]):
655 0
            end_index = (
656
                content_disposition.index(self.__FITS_STRING, start_index + 1) +
657
                len(self.__FITS_STRING))
658 0
            return content_disposition[start_index: end_index]
659 0
        elif (self.__FTZ_STRING in content_disposition[start_index:]):
660 0
            end_index = (
661
                content_disposition.index(self.__FTZ_STRING, start_index + 1) +
662
                len(self.__FTZ_STRING))
663 0
            return content_disposition[start_index: end_index]
664 0
        elif (self.__TAR_STRING in content_disposition[start_index:]):
665 0
            end_index = (
666
                content_disposition.index(self.__TAR_STRING, start_index + 1) +
667
                len(self.__TAR_STRING))
668 0
            return content_disposition[start_index: end_index]
669
        else:
670 0
            raise ValueError("Could not find file name in header. "
671
                             "Content disposition: {}.".format(
672
                                 content_disposition))
673

674 1
    def _extract_file_name_from_url(self, product_url):
675 0
        start_index = product_url.rindex("/") + 1
676 0
        return product_url[start_index:]
677

678 1
    def _query_region_maps(self, coordinates, radius, observation_name,
679
                           get_query_payload, cache):
680 0
        observation_tap_name = (
681
            self._find_observation_tap_table_name(observation_name))
682 0
        query = (
683
            self._build_observation_query(coordinates, radius,
684
                                          self._find_observation_parameters(observation_tap_name)))
685 0
        request_payload = self._create_request_payload(query)
686 0
        if (get_query_payload):
687 0
            return request_payload
688 0
        return self._get_and_parse_from_tap(request_payload, cache)
689

690 1
    def _query_region_catalog(self, coordinates, radius, catalog_name, row_limit,
691
                              get_query_payload, cache):
692 0
        catalog_tap_name = self._find_catalog_tap_table_name(catalog_name)
693 0
        query = self._build_catalog_query(coordinates, radius, row_limit,
694
                                          self._find_catalog_parameters(catalog_tap_name))
695 0
        request_payload = self._create_request_payload(query)
696 0
        if (get_query_payload):
697 0
            return request_payload
698 0
        return self._get_and_parse_from_tap(request_payload, cache)
699

700 1
    def _build_observation_query(self, coordinates, radius, json):
701 0
        raHours, dec = commons.coord_to_radec(coordinates)
702 0
        ra = raHours * 15.0  # Converts to degrees
703 0
        radiusDeg = commons.radius_to_unit(radius, unit='deg')
704

705 0
        select_query = "SELECT DISTINCT "
706

707 0
        metadata = json[self.__METADATA_STRING]
708 0
        metadata_tap_names = ", ".join(["{}".format(entry[self.__TAP_NAME_STRING])
709
                                        for entry in metadata])
710

711 0
        from_query = " FROM {}".format(json[self.__TAP_TABLE_STRING])
712 0
        if (radiusDeg != 0 or json[self.__IS_SURVEY_MISSION_STRING]):
713 0
            if (json[self.__IS_SURVEY_MISSION_STRING]):
714 0
                where_query = (" WHERE 1=CONTAINS(pos, CIRCLE('ICRS', {}, {}, {}));".
715
                               format(ra, dec, radiusDeg))
716
            else:
717 0
                where_query = (" WHERE 1=INTERSECTS(CIRCLE('ICRS', {}, {}, {}), fov);".
718
                               format(ra, dec, radiusDeg))
719
        else:
720 0
            where_query = (" WHERE 1=CONTAINS(POINT('ICRS', {}, {}), fov);".
721
                           format(ra, dec))
722

723 0
        query = "".join([
724
            select_query,
725
            metadata_tap_names,
726
            from_query,
727
            where_query])
728 0
        return query
729

730 1
    def _build_catalog_query(self, coordinates, radius, row_limit, json):
731 0
        raHours, dec = commons.coord_to_radec(coordinates)
732 0
        ra = raHours * 15.0  # Converts to degrees
733 0
        radiusDeg = commons.radius_to_unit(radius, unit='deg')
734

735 0
        select_query = "SELECT "
736 0
        if(row_limit > 0):
737 0
            select_query = "".join([select_query, "TOP {} ".format(row_limit)])
738 0
        elif(not row_limit == -1):
739 0
            raise ValueError("Invalid value of row_limit")
740

741 0
        metadata = json[self.__METADATA_STRING]
742 0
        metadata_tap_names = ", ".join(["{}".format(entry[self.__TAP_NAME_STRING])
743
                                        for entry in metadata])
744

745 0
        from_query = " FROM {}".format(json[self.__TAP_TABLE_STRING])
746 0
        if (radiusDeg == 0):
747 0
            where_query = (" WHERE 1=CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', {}, {}, {}))".
748
                           format(ra,
749
                                  dec,
750
                                  commons.radius_to_unit(
751
                                      self.__MIN_RADIUS_CATALOG_STRING,
752
                                      unit='deg')))
753
        else:
754 0
            where_query = (" WHERE 1=CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', {}, {}, {}))".
755
                           format(ra, dec, radiusDeg))
756 0
        order_by_query = ""
757 0
        if(json[self.__ORDER_BY_STRING] != ""):
758 0
            order_by_query = " ORDER BY {};".format(json[self.__ORDER_BY_STRING])
759

760 0
        query = "".join([select_query, metadata_tap_names, from_query,
761
                        where_query, order_by_query])
762

763 0
        return query
764

765 1
    def _store_query_result_maps(self, query_result, missions, coordinates,
766
                                 radius, get_query_payload, cache):
767 0
        for mission in missions:
768 0
            mission_table = self._query_region_maps(coordinates, radius,
769
                                                    mission, get_query_payload,
770
                                                    cache)
771 0
            if (len(mission_table) > 0):
772 0
                query_result[mission.upper()] = mission_table
773

774 1
    def _store_query_result_catalogs(self, query_result, catalogs, coordinates,
775
                                     radius, row_limit, get_query_payload, cache):
776 0
        for catalog in catalogs:
777 0
            catalog_table = self._query_region_catalog(coordinates, radius,
778
                                                       catalog, row_limit,
779
                                                       get_query_payload, cache)
780 0
            if (len(catalog_table) > 0):
781 0
                query_result[catalog.upper()] = catalog_table
782

783 1
    def _find_observation_parameters(self, mission_name):
784 0
        return self._find_mission_parameters_in_json(mission_name,
785
                                                     self._get_observation_json())
786

787 1
    def _find_catalog_parameters(self, catalog_name):
788 0
        return self._find_mission_parameters_in_json(catalog_name,
789
                                                     self._get_catalogs_json())
790

791 1
    def _find_mission_parameters_in_json(self, mission_tap_name, json):
792 0
        for mission in json:
793 0
            if (mission[self.__TAP_TABLE_STRING] == mission_tap_name):
794 0
                return mission
795 0
        raise ValueError("Input tap name {} not available.".format(mission_tap_name))
796

797 1
    def _find_observation_tap_table_name(self, mission_name):
798 0
        return self._find_mission_tap_table_name(
799
            self._fetch_and_parse_json(self.__OBSERVATIONS_STRING),
800
            mission_name)
801

802 1
    def _find_catalog_tap_table_name(self, mission_name):
803 0
        return self._find_mission_tap_table_name(
804
            self._fetch_and_parse_json(self.__CATALOGS_STRING),
805
            mission_name)
806

807 1
    def _find_mission_tap_table_name(self, json, mission_name):
808 0
        for index in range(len(json)):
809 0
            if (json[index][self.__MISSION_STRING].lower() == mission_name.lower()):
810 0
                return json[index][self.__TAP_TABLE_STRING]
811

812 0
        raise ValueError("Input {} not available.".format(mission_name))
813 0
        return None
814

815 1
    def _get_observation_json(self):
816 0
        return self._fetch_and_parse_json(self.__OBSERVATIONS_STRING)
817

818 1
    def _get_catalogs_json(self):
819 1
        return self._fetch_and_parse_json(self.__CATALOGS_STRING)
820

821 1
    def _fetch_and_parse_json(self, object_name):
822 1
        url = self.URLbase + "/" + object_name
823 1
        response = self._request(
824
            'GET',
825
            url,
826
            cache=False,
827
            headers=self._get_header())
828

829 1
        response.raise_for_status()
830

831 1
        string_response = response.content.decode('utf-8')
832 1
        json_response = json.loads(string_response)
833 1
        return json_response["descriptors"]
834

835 1
    def _json_object_field_to_list(self, json, field_name):
836 1
        response_list = []
837 1
        for index in range(len(json)):
838 1
            response_list.append(json[index][field_name])
839 1
        return response_list
840

841 1
    def _get_json_data_for_mission(self, json, mission):
842 0
        for index in range(len(json)):
843 0
            if(json[index][self.__MISSION_STRING].lower() == mission.lower()):
844 0
                return json[index]
845

846 1
    def _get_tap_observation_id(self, mission):
847 0
        return self._get_json_data_for_mission(self._get_observation_json(), mission)["tapObservationId"]
848

849 1
    def _create_request_payload(self, query):
850 0
        return {'REQUEST': 'doQuery', 'LANG': 'ADQL', 'FORMAT': 'VOTABLE',
851
                'QUERY': query}
852

853 1
    def _get_and_parse_from_tap(self, request_payload, cache):
854 0
        response = self._send_get_request("/tap/sync", request_payload, cache)
855 0
        return self._parse_xml_table(response)
856

857 1
    def _send_get_request(self, url_extension, request_payload, cache):
858 0
        url = self.URLbase + url_extension
859 0
        return self._request('GET',
860
                             url,
861
                             params=request_payload,
862
                             timeout=self.TIMEOUT,
863
                             cache=cache,
864
                             headers=self._get_header())
865

866 1
    def _parse_xml_table(self, response):
867
        # try to parse the result into an astropy.Table, else
868
        # return the raw result with an informative error message.
869 0
        try:
870 0
            tf = six.BytesIO(response.content)
871 0
            vo_table = votable.parse(tf, pedantic=False)
872 0
            first_table = vo_table.get_first_table()
873 0
            table = first_table.to_table(use_names_over_ids=True)
874 0
            return table
875 0
        except Exception as ex:
876 0
            self.response = response
877 0
            self.table_parse_error = ex
878 0
            raise TableParseError(
879
                "Failed to parse ESASky VOTABLE result! The raw response can be "
880
                "found in self.response, and the error in "
881
                "self.table_parse_error.")
882

883 1
    def _get_header(self):
884 1
        user_agent = 'astropy:astroquery.esasky.{vers} {isTest}'.format(
885
            vers=version.version,
886
            isTest=self._isTest)
887 1
        return {'User-Agent': user_agent}
888

889

890 1
ESASky = ESASkyClass()

Read our documentation on viewing source code .

Loading