1
|
|
"""
|
2
|
|
Search functionality for the Gemini archive of observations.
|
3
|
|
|
4
|
|
For questions, contact ooberdorf@gemini.edu
|
5
|
|
"""
|
6
|
|
|
7
|
1
|
import os
|
8
|
|
|
9
|
1
|
from datetime import date
|
10
|
|
|
11
|
1
|
from astropy import log
|
12
|
1
|
from astropy import units
|
13
|
1
|
from astropy.table import Table, MaskedColumn
|
14
|
|
|
15
|
1
|
from astroquery.gemini.urlhelper import URLHelper
|
16
|
1
|
import numpy as np
|
17
|
|
|
18
|
1
|
import logging
|
19
|
|
|
20
|
1
|
from ..query import BaseQuery, QueryWithLogin
|
21
|
1
|
from ..utils.class_or_instance import class_or_instance
|
22
|
1
|
from . import conf
|
23
|
1
|
from ..exceptions import AuthenticationWarning
|
24
|
|
|
25
|
|
|
26
|
1
|
logger = logging.getLogger(__name__)
|
27
|
|
|
28
|
|
|
29
|
1
|
__all__ = ['Observations', 'ObservationsClass'] # specifies what to import
|
30
|
|
|
31
|
|
|
32
|
1
|
__valid_instruments__ = [
|
33
|
|
'GMOS',
|
34
|
|
'GMOS-N',
|
35
|
|
'GMOS-S',
|
36
|
|
'GNIRS',
|
37
|
|
'GRACES',
|
38
|
|
'NIRI',
|
39
|
|
'NIFS',
|
40
|
|
'GSAOI',
|
41
|
|
'F2',
|
42
|
|
'GPI',
|
43
|
|
'NICI',
|
44
|
|
'MICHELLE',
|
45
|
|
'TRECS',
|
46
|
|
'BHROS',
|
47
|
|
'HRWFS',
|
48
|
|
'OSCIR',
|
49
|
|
'FLAMINGOS',
|
50
|
|
'HOKUPAA+QUIRC',
|
51
|
|
'PHOENIX',
|
52
|
|
'TEXES',
|
53
|
|
'ABU',
|
54
|
|
'CIRPASS'
|
55
|
|
]
|
56
|
|
|
57
|
|
|
58
|
1
|
__valid_observation_class__ = [
|
59
|
|
'science',
|
60
|
|
'acq',
|
61
|
|
'progCal',
|
62
|
|
'dayCal',
|
63
|
|
'partnerCal',
|
64
|
|
'acqCal',
|
65
|
|
]
|
66
|
|
|
67
|
1
|
__valid_observation_types__ = [
|
68
|
|
'OBJECT',
|
69
|
|
'BIAS',
|
70
|
|
'DARK',
|
71
|
|
'FLAT',
|
72
|
|
'ARC',
|
73
|
|
'PINHOLE',
|
74
|
|
'RONCHI',
|
75
|
|
'CAL',
|
76
|
|
'FRINGE',
|
77
|
|
'MASK'
|
78
|
|
]
|
79
|
|
|
80
|
1
|
__valid_modes__ = [
|
81
|
|
'imaging',
|
82
|
|
'spectroscopy',
|
83
|
|
'LS',
|
84
|
|
'MOS',
|
85
|
|
'IFS'
|
86
|
|
]
|
87
|
|
|
88
|
1
|
__valid_adaptive_optics__ = [
|
89
|
|
'NOTAO',
|
90
|
|
'AO',
|
91
|
|
'NGS',
|
92
|
|
'LGS'
|
93
|
|
]
|
94
|
|
|
95
|
1
|
__valid_raw_reduced__ = [
|
96
|
|
'RAW',
|
97
|
|
'PREPARED',
|
98
|
|
'PROCESSED_BIAS',
|
99
|
|
'PROCESSED_FLAT',
|
100
|
|
'PROCESSED_FRINGE',
|
101
|
|
'PROCESSED_ARC'
|
102
|
|
]
|
103
|
|
|
104
|
|
|
105
|
1
|
class ObservationsClass(QueryWithLogin):
|
106
|
|
|
107
|
1
|
server = conf.server
|
108
|
1
|
url_helper = URLHelper(server)
|
109
|
|
|
110
|
1
|
def __init__(self, *args):
|
111
|
|
"""
|
112
|
|
Query class for observations in the Gemini archive.
|
113
|
|
|
114
|
|
This class provides query capabilities against the gemini archive. Queries
|
115
|
|
can be done by cone search, by name, or by a set of criteria.
|
116
|
|
"""
|
117
|
1
|
super().__init__()
|
118
|
|
|
119
|
1
|
def _login(self, username, password):
|
120
|
|
"""
|
121
|
|
Login to the Gemini Archive website.
|
122
|
|
|
123
|
|
This method will authenticate the session as a particular user. This may give you access
|
124
|
|
to additional information or access based on your credentials
|
125
|
|
|
126
|
|
Parameters
|
127
|
|
----------
|
128
|
|
username : str
|
129
|
|
The username to login as
|
130
|
|
password : str
|
131
|
|
The password for the given account
|
132
|
|
"""
|
133
|
0
|
params = dict(username=username, password=password)
|
134
|
0
|
r = self._session.request('POST', 'https://archive.gemini.edu/login/', params=params)
|
135
|
0
|
if b'<P>Welcome, you are sucessfully logged in' not in r.content:
|
136
|
0
|
logger.error('Unable to login, please check your credentials')
|
137
|
0
|
return False
|
138
|
0
|
return True
|
139
|
|
|
140
|
1
|
@class_or_instance
|
141
|
1
|
def query_region(self, coordinates, radius=0.3*units.deg):
|
142
|
|
"""
|
143
|
|
search for Gemini observations by target on the sky.
|
144
|
|
|
145
|
|
Given a sky position and radius, returns a `~astropy.table.Table` of Gemini observations.
|
146
|
|
|
147
|
|
Parameters
|
148
|
|
----------
|
149
|
|
coordinates : str or `~astropy.coordinates` object
|
150
|
|
The target around which to search. It may be specified as a
|
151
|
|
string or as the appropriate `~astropy.coordinates` object.
|
152
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
153
|
|
Default 0.3 degrees.
|
154
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
155
|
|
appropriate `~astropy.units.Quantity` object from
|
156
|
|
`~astropy.units` may also be used. Defaults to 0.3 deg.
|
157
|
|
|
158
|
|
Returns
|
159
|
|
-------
|
160
|
|
response : `~astropy.table.Table`
|
161
|
|
"""
|
162
|
1
|
return self.query_criteria(coordinates=coordinates, radius=radius)
|
163
|
|
|
164
|
1
|
@class_or_instance
|
165
|
1
|
def query_object(self, objectname, radius=0.3*units.deg):
|
166
|
|
"""
|
167
|
|
search for Gemini observations by target on the sky.
|
168
|
|
|
169
|
|
Given an object name and optional radius, returns a `~astropy.table.Table` of Gemini observations.
|
170
|
|
|
171
|
|
Parameters
|
172
|
|
----------
|
173
|
|
objectname : str
|
174
|
|
The name of an object to search for. This attempts to resolve the object
|
175
|
|
by name and do a search on that area of the sky. This does not handle
|
176
|
|
moving targets.
|
177
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
178
|
|
Default 0.3 degrees.
|
179
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
180
|
|
appropriate `~astropy.units.Quantity` object from
|
181
|
|
`~astropy.units` may also be used. Defaults to 0.3 deg.
|
182
|
|
|
183
|
|
Returns
|
184
|
|
-------
|
185
|
|
response : `~astropy.table.Table`
|
186
|
|
"""
|
187
|
0
|
return self.query_criteria(objectname=objectname, radius=radius)
|
188
|
|
|
189
|
1
|
@class_or_instance
|
190
|
1
|
def query_criteria(self, *rawqueryargs, coordinates=None, radius=0.3*units.deg, pi_name=None, program_id=None, utc_date=None,
|
191
|
|
instrument=None, observation_class=None, observation_type=None, mode=None,
|
192
|
|
adaptive_optics=None, program_text=None, objectname=None, raw_reduced=None,
|
193
|
|
orderby=None, **rawquerykwargs):
|
194
|
|
"""
|
195
|
|
search a variety of known parameters against the Gemini observations.
|
196
|
|
|
197
|
|
Given various criteria, search the Gemini archive for matching observations. Note that
|
198
|
|
``rawqueryargs`` and ``rawquerykwargs`` will pick up additional positional and key=value
|
199
|
|
arguments and pass then on to the raw query as is.
|
200
|
|
|
201
|
|
Parameters
|
202
|
|
----------
|
203
|
|
coordinates : str or `~astropy.coordinates` object
|
204
|
|
The target around which to search. It may be specified as a
|
205
|
|
string or as the appropriate `~astropy.coordinates` object.
|
206
|
|
radius : str or `~astropy.units.Quantity` object, optional
|
207
|
|
Default 0.3 degrees.
|
208
|
|
The string must be parsable by `~astropy.coordinates.Angle`. The
|
209
|
|
appropriate `~astropy.units.Quantity` object from
|
210
|
|
`~astropy.units` may also be used. Defaults to 0.3 deg.
|
211
|
|
pi_name : str, optional
|
212
|
|
Default None.
|
213
|
|
Can be used to search for data by the PI's name.
|
214
|
|
program_id : str, optional
|
215
|
|
Default None.
|
216
|
|
Can be used to match on program ID
|
217
|
|
utc_date : date or (date,date) tuple, optional
|
218
|
|
Default None.
|
219
|
|
Can be used to search for observations on a particular day or range of days (inclusive).
|
220
|
|
instrument : str, optional
|
221
|
|
Can be used to search for a particular instrument. Valid values are:
|
222
|
|
'GMOS',
|
223
|
|
'GMOS-N',
|
224
|
|
'GMOS-S',
|
225
|
|
'GNIRS',
|
226
|
|
'GRACES',
|
227
|
|
'NIRI',
|
228
|
|
'NIFS',
|
229
|
|
'GSAOI',
|
230
|
|
'F2',
|
231
|
|
'GPI',
|
232
|
|
'NICI',
|
233
|
|
'MICHELLE',
|
234
|
|
'TRECS',
|
235
|
|
'BHROS',
|
236
|
|
'HRWFS',
|
237
|
|
'OSCIR',
|
238
|
|
'FLAMINGOS',
|
239
|
|
'HOKUPAA+QUIRC',
|
240
|
|
'PHOENIX',
|
241
|
|
'TEXES',
|
242
|
|
'ABU',
|
243
|
|
'CIRPASS'
|
244
|
|
observation_class : str, optional
|
245
|
|
Specifies the class of observations to search for. Valid values are:
|
246
|
|
'science',
|
247
|
|
'acq',
|
248
|
|
'progCal',
|
249
|
|
'dayCal',
|
250
|
|
'partnerCal',
|
251
|
|
'acqCal'
|
252
|
|
observation_type : str, optional
|
253
|
|
Search for a particular type of observation. Valid values are:
|
254
|
|
'OBJECT',
|
255
|
|
'BIAS',
|
256
|
|
'DARK',
|
257
|
|
'FLAT',
|
258
|
|
'ARC',
|
259
|
|
'PINHOLE',
|
260
|
|
'RONCHI',
|
261
|
|
'CAL',
|
262
|
|
'FRINGE',
|
263
|
|
'MASK'
|
264
|
|
mode : str, optional
|
265
|
|
The mode of the observation. Valid values are:
|
266
|
|
'imaging',
|
267
|
|
'spectroscopy',
|
268
|
|
'LS',
|
269
|
|
'MOS',
|
270
|
|
'IFS'
|
271
|
|
adaptive_optics : str, optional
|
272
|
|
Specify the presence of adaptive optics. Valid values are:
|
273
|
|
'NOTAO',
|
274
|
|
'AO',
|
275
|
|
'NGS',
|
276
|
|
'LGS'
|
277
|
|
program_text : str, optional
|
278
|
|
Specify text in the information about the program. This is free form text.
|
279
|
|
objectname : str, optional
|
280
|
|
Give the name of the target.
|
281
|
|
raw_reduced : str, optional
|
282
|
|
Indicate the raw or reduced status of the observations to search for. Valid values are:
|
283
|
|
'RAW',
|
284
|
|
'PREPARED',
|
285
|
|
'PROCESSED_BIAS',
|
286
|
|
'PROCESSED_FLAT',
|
287
|
|
'PROCESSED_FRINGE',
|
288
|
|
'PROCESSED_ARC'
|
289
|
|
orderby : str, optional
|
290
|
|
Indicates how the results should be sorted. Values should be like the ones used
|
291
|
|
in the archive website when sorting a column. For example, ``data_label_desc`` would
|
292
|
|
sort by the data label in descending order.
|
293
|
|
rawqueryargs : list, optional
|
294
|
|
Additional arguments will be passed down to the raw query. This covers any
|
295
|
|
additional parameters that would end up as '/parametervalue/' in the URL to the archive
|
296
|
|
webservice.
|
297
|
|
rawquerykwargs : dict, optional
|
298
|
|
Additional key/value arguments will also be passed down to the raw query. This covers
|
299
|
|
any parameters that would end up as '/key=value/' in the URL to the archive webservice.
|
300
|
|
|
301
|
|
Returns
|
302
|
|
-------
|
303
|
|
response : `~astropy.table.Table`
|
304
|
|
|
305
|
|
Raises
|
306
|
|
------
|
307
|
|
ValueError: passed value is not recognized for the given field, see message for details
|
308
|
|
"""
|
309
|
|
|
310
|
|
# Build parameters into raw query
|
311
|
|
#
|
312
|
|
# This consists of a set of unnamed arguments, args, and key/value pairs, kwargs
|
313
|
|
|
314
|
|
# These will hold the passed freeform parameters plus the explicit criteria
|
315
|
|
# for our eventual call to the raw query method
|
316
|
1
|
args = list()
|
317
|
1
|
kwargs = dict()
|
318
|
|
|
319
|
|
# Copy the incoming set of free-form arguments
|
320
|
1
|
if rawqueryargs:
|
321
|
0
|
for arg in rawqueryargs:
|
322
|
0
|
args.append(arg)
|
323
|
1
|
if rawquerykwargs:
|
324
|
0
|
for (k, v) in rawquerykwargs.items():
|
325
|
0
|
kwargs[k] = v
|
326
|
|
|
327
|
|
# Now consider the canned criteria
|
328
|
1
|
if radius is not None:
|
329
|
1
|
kwargs["radius"] = radius
|
330
|
1
|
if coordinates is not None:
|
331
|
1
|
kwargs["coordinates"] = coordinates
|
332
|
1
|
if pi_name is not None:
|
333
|
0
|
kwargs["PIname"] = pi_name
|
334
|
1
|
if program_id is not None:
|
335
|
1
|
kwargs["progid"] = program_id.upper()
|
336
|
1
|
if utc_date is not None:
|
337
|
0
|
if isinstance(utc_date, date):
|
338
|
0
|
args.append(utc_date.strftime("YYYYMMdd"))
|
339
|
0
|
elif isinstance(utc_date, tuple):
|
340
|
0
|
if len(utc_date) != 2:
|
341
|
0
|
raise ValueError("utc_date tuple should have two values")
|
342
|
0
|
if not isinstance(utc_date[0], date) or not isinstance(utc_date[1], date):
|
343
|
0
|
raise ValueError("utc_date tuple should have date values in it")
|
344
|
0
|
args.append("%s-%s" % utc_date[0].strftime("YYYYMMdd"), utc_date[1].strftime("YYYYMMdd"))
|
345
|
1
|
if instrument is not None:
|
346
|
1
|
if instrument.upper() not in __valid_instruments__:
|
347
|
0
|
raise ValueError("Unrecognized instrument: %s" % instrument)
|
348
|
1
|
args.append(instrument)
|
349
|
1
|
if observation_class is not None:
|
350
|
0
|
if observation_class not in __valid_observation_class__:
|
351
|
0
|
raise ValueError("Unrecognized observation class: %s" % observation_class)
|
352
|
0
|
args.append(observation_class)
|
353
|
1
|
if observation_type is not None:
|
354
|
1
|
if observation_type not in __valid_observation_types__:
|
355
|
0
|
raise ValueError("Unrecognized observation type: %s" % observation_type)
|
356
|
1
|
args.append(observation_type)
|
357
|
1
|
if mode is not None:
|
358
|
0
|
if mode not in __valid_modes__:
|
359
|
0
|
raise ValueError("Unrecognized mode: %s" % mode)
|
360
|
0
|
args.append(mode)
|
361
|
1
|
if adaptive_optics is not None:
|
362
|
0
|
if adaptive_optics not in __valid_adaptive_optics__:
|
363
|
0
|
raise ValueError("Unrecognized adaptive optics: %s" % adaptive_optics)
|
364
|
0
|
args.append(adaptive_optics)
|
365
|
1
|
if program_text is not None:
|
366
|
0
|
kwargs["ProgramText"] = program_text
|
367
|
1
|
if objectname is not None:
|
368
|
0
|
kwargs["object"] = objectname
|
369
|
1
|
if raw_reduced is not None:
|
370
|
0
|
if raw_reduced not in __valid_raw_reduced__:
|
371
|
0
|
raise ValueError("Unrecognized raw/reduced setting: %s" % raw_reduced)
|
372
|
0
|
args.append(raw_reduced)
|
373
|
1
|
if orderby is not None:
|
374
|
0
|
kwargs["orderby"] = orderby
|
375
|
|
|
376
|
1
|
return self.query_raw(*args, **kwargs)
|
377
|
|
|
378
|
1
|
@class_or_instance
|
379
|
|
def query_raw(self, *args, **kwargs):
|
380
|
|
"""
|
381
|
|
perform flexible query against Gemini observations
|
382
|
|
|
383
|
|
This is a more flexible query method. This method will do special handling for
|
384
|
|
coordinates and radius if present in kwargs. However, for the remaining arguments
|
385
|
|
it assumes all of args are useable as query path elements. For kwargs, it assumes
|
386
|
|
all of the elements can be passed as name=value within the query path to Gemini.
|
387
|
|
|
388
|
|
This method does not do any validation checking or attempt to interperet the
|
389
|
|
values being passed, aside from coordinates and radius.
|
390
|
|
|
391
|
|
This method is most useful when the query_criteria and query_region do not
|
392
|
|
meet your needs and you can build the appropriate search in the website. When
|
393
|
|
you see the URL that is generated by the archive, you can translate that into
|
394
|
|
an equivalent python call with this method. For example, if the URL in the
|
395
|
|
website is:
|
396
|
|
|
397
|
|
https://archive.gemini.edu/searchform/RAW/cols=CTOWEQ/notengineering/GMOS-N/PIname=Hirst/NotFail
|
398
|
|
|
399
|
|
You can disregard NotFail, cols=x and notengineering. You would run this query as
|
400
|
|
|
401
|
|
query_raw('GMOS-N', PIname='Hirst')
|
402
|
|
|
403
|
|
Parameters
|
404
|
|
----------
|
405
|
|
args :
|
406
|
|
The list of parameters to be passed via the query path to the webserver
|
407
|
|
kwargs :
|
408
|
|
The dictionary of parameters to be passed by name=value within the query
|
409
|
|
path to the webserver. The ``orderby`` key value pair has a special
|
410
|
|
intepretation and is appended as a query parameter like the one used
|
411
|
|
in the archive website for sorting results.
|
412
|
|
|
413
|
|
Returns
|
414
|
|
-------
|
415
|
|
response : `~astropy.table.Table`
|
416
|
|
"""
|
417
|
1
|
url = self.url_helper.build_url(*args, **kwargs)
|
418
|
|
|
419
|
1
|
response = self._request(method="GET", url=url, data={}, timeout=180, cache=False)
|
420
|
|
|
421
|
1
|
js = response.json()
|
422
|
1
|
return _gemini_json_to_table(js)
|
423
|
|
|
424
|
1
|
def get_file(self, filename, *, download_dir='.', timeout=None):
|
425
|
|
"""
|
426
|
|
Download the requested file to the current directory
|
427
|
|
|
428
|
|
filename : str
|
429
|
|
Name of the file to download
|
430
|
|
download_dir : str, optional
|
431
|
|
Name of the directory to download to
|
432
|
|
timeout : int, optional
|
433
|
|
Timeout of the request in milliseconds
|
434
|
|
"""
|
435
|
0
|
url = "https://archive.gemini.edu/file/%s" % filename
|
436
|
0
|
local_filepath = os.path.join(download_dir, filename)
|
437
|
0
|
self._download_file(url=url, local_filepath=local_filepath, timeout=timeout)
|
438
|
|
|
439
|
|
|
440
|
1
|
def _gemini_json_to_table(json):
|
441
|
|
"""
|
442
|
|
takes a JSON object as returned from the Gemini archive webserver and turns it into an `~astropy.table.Table`
|
443
|
|
|
444
|
|
Parameters
|
445
|
|
----------
|
446
|
|
json : dict
|
447
|
|
A JSON object from the Gemini archive webserver
|
448
|
|
|
449
|
|
Returns
|
450
|
|
-------
|
451
|
|
response : `~astropy.table.Table`
|
452
|
|
"""
|
453
|
|
|
454
|
1
|
data_table = Table(masked=True)
|
455
|
|
|
456
|
1
|
for key in __keys__:
|
457
|
1
|
col_data = np.array([obj.get(key, None) for obj in json])
|
458
|
|
|
459
|
1
|
atype = str
|
460
|
|
|
461
|
1
|
col_mask = np.equal(col_data, None)
|
462
|
1
|
data_table.add_column(MaskedColumn(col_data.astype(atype), name=key, mask=col_mask))
|
463
|
|
|
464
|
1
|
return data_table
|
465
|
|
|
466
|
|
|
467
|
1
|
__keys__ = ["exposure_time",
|
468
|
|
"detector_roi_setting",
|
469
|
|
"detector_welldepth_setting",
|
470
|
|
"telescope",
|
471
|
|
"mdready",
|
472
|
|
"requested_bg",
|
473
|
|
"engineering",
|
474
|
|
"cass_rotator_pa",
|
475
|
|
"ut_datetime",
|
476
|
|
"file_size",
|
477
|
|
"types",
|
478
|
|
"requested_wv",
|
479
|
|
"detector_readspeed_setting",
|
480
|
|
"size",
|
481
|
|
"laser_guide_star",
|
482
|
|
"observation_id",
|
483
|
|
"science_verification",
|
484
|
|
"raw_cc",
|
485
|
|
"filename",
|
486
|
|
"instrument",
|
487
|
|
"reduction",
|
488
|
|
"camera",
|
489
|
|
"ra",
|
490
|
|
"detector_binning",
|
491
|
|
"lastmod",
|
492
|
|
"wavelength_band",
|
493
|
|
"data_size",
|
494
|
|
"mode",
|
495
|
|
"raw_iq",
|
496
|
|
"airmass",
|
497
|
|
"elevation",
|
498
|
|
"data_label",
|
499
|
|
"requested_iq",
|
500
|
|
"object",
|
501
|
|
"requested_cc",
|
502
|
|
"program_id",
|
503
|
|
"file_md5",
|
504
|
|
"central_wavelength",
|
505
|
|
"raw_wv",
|
506
|
|
"compressed",
|
507
|
|
"filter_name",
|
508
|
|
"detector_gain_setting",
|
509
|
|
"path",
|
510
|
|
"observation_class",
|
511
|
|
"qa_state",
|
512
|
|
"observation_type",
|
513
|
|
"calibration_program",
|
514
|
|
"md5",
|
515
|
|
"adaptive_optics",
|
516
|
|
"name",
|
517
|
|
"focal_plane_mask",
|
518
|
|
"data_md5",
|
519
|
|
"raw_bg",
|
520
|
|
"disperser",
|
521
|
|
"wavefront_sensor",
|
522
|
|
"gcal_lamp",
|
523
|
|
"detector_readmode_setting",
|
524
|
|
"phot_standard",
|
525
|
|
"local_time",
|
526
|
|
"spectroscopy",
|
527
|
|
"azimuth",
|
528
|
|
"release",
|
529
|
|
"dec"]
|
530
|
|
|
531
|
1
|
Observations = ObservationsClass()
|