采用python 批量下载ocean color中的卫星数据
1.安装python
关于python安装，可以参考网上安装步骤，下载python包，自行安装；
2.数据准备工作
(1)注册一个earthdata账号，建议使用google账号，方便通过。
注册时的网站参考链接：https://oceancolor.gsfc.nasa.gov/registration/
(2)注册完成后获得一个AppKey，获得网站链接： https://oceandata.sci.gsfc.nasa.gov/appkey/`
(3)下载一个python脚本，用于后面数据下载运行，后面附录中有一个样例脚本obdaac_download.py
(4)根据自己的需求，预定数据，预定数据链接：https://oceancolor.gsfc.nasa.gov/l3/， 之后邮件会收到一个预定清单；
(5)使用脚本下载数据，直接使用cmd下载数据，
usage：obdaac_download [-h] [-v] [–filelist FILELIST] [–http_manifest HTTP_MANIFEST] [–odir ODIR] [–uncompress] [–force]
[filename]
- positional arguments:
filename - name of the file (or the URL of the file) to retrieve
- optional arguments:
-h, --help - show this help message and exit
-v, --verbose - print status messages
–filelist FILELIST - file containing list of filenames to retrieve, one per line
–http_manifest HTTP_MANIFEST - URL to http_manifest file for OB.DAAC data order
–odir ODIR - full path to desired output directory; defaults to current working directory.
–uncompress - uncompress the retrieved files (if compressed)
–appkey APPKEY - value of the users application key
–force - force download even if file already exists locally
case：
python obdaac_download.py -v
–http_manifest
–odir D:\data\satelite_data\NUIST_data
–uncompress --force
–appkey ac22f
大家可以参考
import argparse
import hashlib
import os
import re
import sys
import subprocess
import logging
import requests
from requests.adapters import HTTPAdapter
from datetime import datetime
import time
import textwrap
from urllib.parse import urlparse
from pathlib import Path
DEFAULT_CHUNK_SIZE = 131072
BLOCKSIZE = 65536
# requests session object used to keep connections around
obpgSession = None
def getSession(verbose=0, ntries=5):
global obpgSession
if not obpgSession:
# turn on debug statements for requests
if verbose > 1:
print("Session started")
logging.basicConfig(level=logging.DEBUG)
obpgSession = requests.Session()
obpgSession.mount('https://', HTTPAdapter(max_retries=ntries))
else:
if verbose > 1:
print("Reusing existing session")
return obpgSession
def isRequestAuthFailure(req) :
ctype = req.headers.get('Content-Type')
if ctype and ctype.startswith('text/html'):
if "<title>Earthdata Login</title>" in req.text:
return True
return False
def httpdl(server, request, localpath='.', outputfilename=None, ntries=5,
uncompress=False, timeout=30., verbose=0, force_download=False,
chunk_size=DEFAULT_CHUNK_SIZE):
status = 0
urlStr = 'https://' + server + request
global obpgSession
localpath = Path(localpath)
getSession(verbose=verbose, ntries=ntries)
modified_since = None
headers = {}
if not force_download:
if outputfilename:
ofile = localpath / outputfilename
modified_since = get_file_time(ofile)
else:
rpath = Path(request.rstrip())
if 'requested_files' in request:
rpath = Path(request.rstrip().split('?')[0])
ofile = localpath / rpath.name
if re.search(r'(?<=\?)(\w+)', ofile.name):
ofile = Path(ofile.name.split('?')[0])
modified_since = get_file_time(ofile)
if modified_since:
headers = {"If-Modified-Since":modified_since.strftime("%a, %d %b %Y %H:%M:%S GMT")}
with obpgSession.get(urlStr, stream=True, timeout=timeout, headers=headers) as req:
if req.status_code != 200:
status = req.status_code
elif isRequestAuthFailure(req):
status = 401
else:
if not Path.exists(localpath):
os.umask(0o02)
Path.mkdir(localpath, mode=0o2775, parents=True)
if not outputfilename:
cd = req.headers.get('Content-Disposition')
if cd:
outputfilename = re.findall("filename=(.+)", cd)[0]
else:
outputfilename = urlStr.split('/')[-1]
ofile = localpath / outputfilename
# This is here just in case we didn't get a 304 when we should have...
download = True
if 'last-modified' in req.headers:
remote_lmt = req.headers['last-modified']
remote_ftime = datetime.strptime(remote_lmt, "%a, %d %b %Y %H:%M:%S GMT").replace(tzinfo=None)
if modified_since and not force_download:
if (remote_ftime - modified_since).total_seconds() < 0:
download = False
if verbose:
print("Skipping download of %s" % outputfilename)
if download:
total_length = req.headers.get('content-length')
length_downloaded = 0
total_length = int(total_length)
if verbose >0:
print("Downloading %s (%8.2f MBs)" % (outputfilename,total_length /1024/1024))
with open(ofile, 'wb') as fd:
for chunk in req.iter_content(chunk_size=chunk_size):
if chunk: # filter out keep-alive new chunks
length_downloaded += len(chunk)
fd.write(chunk)
if verbose > 0:
percent_done = int(50 * length_downloaded / total_length)
sys.stdout.write("\r[%s%s]" % ('=' * percent_done, ' ' * (50-percent_done)))
sys.stdout.flush()
if uncompress:
if ofile.suffix in {'.Z', '.gz', '.bz2'}:
if verbose:
print("\nUncompressing {}".format(ofile))
compressStatus = uncompressFile(ofile)
if compressStatus:
status = compressStatus
else:
status = 0
if verbose:
print("\n...Done")
return status
`
`def uncompressFile(compressed_file):
"""
uncompress file
compression methods:
bzip2
gzip
UNIX compress
"""
compProg = {".gz": "gunzip -f ", ".Z": "gunzip -f ", ".bz2": "bunzip2 -f "}
exten = Path(compressed_file).suffix
unzip = compProg[exten]
p = subprocess.Popen(unzip + str(compressed_file.resolve()), shell=True)
status = os.waitpid(p.pid, 0)[1]
if status:
print("Warning! Unable to decompress %s" % compressed_file)
return status
else:
return 0
def get_file_time(localFile):
ftime = None
localFile = Path(localFile)
if not Path.is_file(localFile):
while localFile.suffix in {'.Z', '.gz', '.bz2'}:
localFile = localFile.with_suffix('')
if Path.is_file(localFile):
ftime = datetime.fromtimestamp(localFile.stat().st_mtime)
return ftime
def compare_checksum(filepath,checksum):
hasher = hashlib.sha1()
with open(filepath, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
if hasher.hexdigest() == checksum:
return False
else:
return True
def retrieveURL(request,localpath='.', uncompress=False, verbose=0,force_download=False, appkey=False, checksum=False):
if args.verbose:
print("Retrieving %s" % request.rstrip())
server = "oceandata.sci.gsfc.nasa.gov"
parsedRequest = urlparse(request)
netpath = parsedRequest.path
if parsedRequest.netloc:
server = parsedRequest.netloc
else:
if not re.match(".*getfile",netpath):
netpath = '/ob/getfile/' + netpath
joiner = '?'
if (re.match(".*getfile",netpath)) and appkey:
netpath = netpath + joiner +'appkey=' + appkey
joiner = '&'
if parsedRequest.query:
netpath = netpath + joiner + parsedRequest.query
status = httpdl(server, netpath, localpath=localpath, uncompress=uncompress, verbose=verbose,force_download=force_download)
if checksum and not uncompress:
cksumURL = 'https://'+server + '/checkdata/' + parsedRequest.path
dnldfile = localpath / parsedRequest.path
if compare_checksum(dnldfile,requests.get(cksumURL).text):
print("The file %s failed checksum test" % parsedRequest.path)
status = 1
return status
if __name__ == "__main__":
# parse command line
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
description='Download files archived at the OB.DAAC',
epilog=textwrap.dedent('''
Provide one of either filename, --filelist or --http_manifest.
NOTE: For authentication, a valid .netrc file in the user home ($HOME) directory\nor a valid appkey is required.
Example .netrc:
machine urs.earthdata.nasa.gov login USERNAME password PASSWD\n
An appkey can be obtained from:
https://oceandata.sci.gsfc.nasa.gov/appkey/
'''
))
parser.add_argument('-v', '--verbose', help='print status messages',
action='count',default=0)
parser.add_argument('filename', nargs='?', help='name of the file (or the URL of the file) to retreive')
parser.add_argument('--filelist',
help='file containing list of filenames to retreive, one per line')
parser.add_argument('--http_manifest',
help='URL to http_manifest file for OB.DAAC data order')
parser.add_argument('--odir',
help='full path to desired output directory; \ndefaults to current working directory: %s' % Path.cwd(),
default=Path.cwd())
parser.add_argument('--uncompress',action="store_true",
help='uncompress the retrieved files (if compressed)',
default=False)
parser.add_argument('--checksum',action="store_true",
help='compare retrieved file checksum; cannot be used with --uncompress',
default=False)
parser.add_argument('--failed',help='filename to contain list of files that failed to be retrieved')
parser.add_argument('--appkey',help='value of the users application key')
parser.add_argument('--force',action='store_true',
help='force download even if file already exists locally',
default=False)
args = parser.parse_args()
filelist = []
if args.http_manifest:
status = retrieveURL(args.http_manifest,verbose=args.verbose,force_download=True,appkey=args.appkey)
if status:
print("There was a problem retrieving %s (received status %d)" % (args.http_manifest,status))
sys.exit("Bailing out...")
else:
with open('http_manifest.txt') as flist:
for filename in flist:
filelist.append(filename.rstrip())
elif args.filename:
filelist.append(args.filename)
elif args.filelist:
with open(os.path.expandvars(args.filelist)) as flist:
for filename in flist:
filelist.append(os.path.expandvars(filename.rstrip()))
if not len(filelist):
parser.print_usage()
sys.exit("Please provide a filename (or list file) to retrieve")
if args.uncompress and args.checksum:
parser.print_usage()
sys.exit("--uncompress is incompatible with --checksum")
outpath = Path.resolve(Path.expanduser(Path(os.path.expandvars(args.odir))))
if args.verbose:
print("Output directory: %s" % outpath)
failed = None
if args.failed:
failed = open(args.failed, 'w')
for request in filelist:
status = retrieveURL(request,localpath=outpath, uncompress=args.uncompress,
verbose=args.verbose,force_download=args.force,
appkey=args.appkey,checksum=args.checksum)
if status:
if status == 304:
if args.verbose:
print("%s is not newer than local copy, skipping download" % request)
else:
print("There was a problem retrieving %s (received status %d)" % (request,status))
if failed:
failed.write(request)
failed.write("\n")
if failed:
failed.close()`
参考文章链接：https://oceancolor.gsfc.nasa.gov/data/download_methods/