1
# Licensed under a 3-clause BSD style license - see LICENSE.rst
2 1
import re
3 1
import string
4 1
import os
5 1
import gzip
6

7 1
from six import StringIO
8 1
import astropy.io.fits as fits
9 1
from .commons import get_readable_fileobj
10

11 1
__all__ = ['download_list_of_fitsfiles']
12

13 1
__doctest_skip__ = ["download_list_of_fitsfiles"]
14

15 1
whitespace_re = re.compile(r"\s")
16 1
valid_chars = "-_.()%s%s" % (string.ascii_letters, string.digits)
17

18

19 1
def validify_filename(filestr):
20
    """ Remove invalid characters from a file string """
21 0
    filestr = filestr.strip()
22 0
    filestr = whitespace_re.sub("_", filestr)
23

24
    # strip out invalid characters
25 0
    filestr = "".join([c for c in filestr if c in valid_chars])
26 0
    return filestr
27

28

29 1
def download_list_of_fitsfiles(linklist, output_directory=None,
30
                               output_prefix=None, save=False,
31
                               overwrite=False, verbose=False,
32
                               output_coord_format=None,
33
                               filename_header_keywords=None,
34
                               include_input_filename=True):
35
    """
36
    Given a list of file URLs, download them and (optionally) rename them.
37

38
    Examples
39
    --------
40

41
    >>> linklist = ['http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH00.fits',
42
    ...             'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH01.fits',
43
    ...             'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_SC00.fits']
44
    >>> download_list_of_fitsfiles(linklist,
45
    ...     output_directory='fermi_m31',
46
    ...     output_prefix='FermiLAT',
47
    ...     save=True,
48
    ...     overwrite=False,
49
    ...     verbose=True,
50
    ...     output_coord_format=None, # FITS tables don't have crval/crpix, good one is: "%08.3g%+08.3g",
51
    ...     filename_header_keywords=None, # couldn't find any useful ones
52
    ...     include_input_filename=True)
53
    """
54
    # Loop through links and retrieve FITS images
55 0
    images = {}
56 0
    for link in linklist:
57

58 0
        if output_directory is None:
59 0
            output_directory = ""
60 0
        elif output_directory[-1] != "/":
61 0
            output_directory += "/"
62 0
            if not os.path.exists(output_directory):
63 0
                os.mkdir(output_directory)
64

65 0
        with get_readable_fileobj(link, cache=True) as f:
66 0
            results = f.read()
67 0
        S = StringIO(results)
68

69 0
        try:
70
            # try to open as a fits file
71 0
            fitsfile = fits.open(S, ignore_missing_end=True)
72 0
        except IOError:
73
            # if that fails, try to open as a gzip'd fits file
74
            # have to rewind to the start
75 0
            S.seek(0)
76 0
            G = gzip.GzipFile(fileobj=S)
77 0
            fitsfile = fits.open(G, ignore_missing_end=True)
78

79
        # Get Multiframe ID from the header
80 0
        images[link] = fitsfile
81

82 0
        if save:
83 0
            h0 = fitsfile[0].header
84

85 0
            if filename_header_keywords:  # is not None or empty
86 0
                nametxt = "_".join([validify_filename(str(h0[key]))
87
                                    for key in filename_header_keywords])
88
            else:
89 0
                nametxt = ""
90

91 0
            if output_coord_format:
92 0
                lon = h0['CRVAL1']
93 0
                lat = h0['CRVAL2']
94

95 0
                try:
96 0
                    coordstr = output_coord_format.format(lon, lat)
97 0
                except TypeError:
98 0
                    coordstr = output_coord_format % (lon, lat)
99 0
                nametxt += "_" + coordstr
100

101 0
            if include_input_filename:
102 0
                filename_root = os.path.split(link)[1]
103
            else:
104 0
                filename_root = ""
105

106 0
            savename = output_prefix if output_prefix else ""
107 0
            savename += nametxt
108 0
            savename += "_" + filename_root
109

110
            # Set final directory and file names
111 0
            final_file = output_directory + savename
112

113 0
            if verbose:
114 0
                print("Saving file %s" % final_file)
115

116 0
            try:
117 0
                fitsfile.writeto(final_file, clobber=overwrite)
118 0
            except IOError:
119 0
                print("Skipped writing file {0} because it exists "
120
                      "and overwrite=False".format(final_file))
121

122 0
    return images

Read our documentation on viewing source code .

Loading