VIIRSdownloader/VIIRSdownloader.py

# -*- coding: utf-8 -*-
"""
Downloads the actual active fire csv from
https://firms.modaps.eosdis.nasa.gov/data/active_fire/noaa-20-viirs-c2/csv/J1_VIIRS_C2_Europe_24h.csv
and displays it on a map.
Description of data displayed:
https://www.earthdata.nasa.gov/learn/find-data/near-real-time/firms/vnp14imgtdlnrt#ed-viirs-375m-attributes
@author: faraway
"""
import argparse
import getpass
import re
import os.path
import pickle
import datetime as dt
import webbrowser
import requests
import geopandas as gpd
import folium
from shapely.geometry import Point
from tqdm import tqdm
from sentinelsat.sentinel import SentinelAPI, geojson_to_wkt
from geopy.geocoders import Nominatim
from geopy.extra.rate_limiter import RateLimiter
from branca.element import Element, JavascriptLink, CssLink


def check_file_format(args_inputfile):
    pattern = re.compile(r'local_VIIRS_data-\d{4}-\d{2}-\d{2}.zip')
    if not re.match(pattern, args_inputfile):
        raise argparse.ArgumentTypeError
    return args_inputfile


def get_address(geo_data_frame, date):
    address_file = 'adressdump-' + date
    if not os.path.exists(address_file):
        address_list = []
        geolocator = Nominatim(user_agent='blub browser')
        reverse = RateLimiter(geolocator.reverse, min_delay_seconds=5)
        p_bar = tqdm(total=len(geo_data_frame))
        for r in geo_data_frame.iterrows():
            address_list.append(
                reverse(r[1]['geometry'].coords[0], language='en'))
            p_bar.update(1)
        p_bar.close()
        try:
            with open(address_file, 'wb') as file:
                pickle.dump(address_list, file)
        except Exception as e:
            SystemExit(e)
    else:
        try:
            with open(address_file, 'rb') as file:
                address_list = pickle.load(file)
        except Exception as e:
            SystemExit(e)
    return address_list


def render_map(args, in_ukraine, address_list, date):
    src1 = (r'https://gibs.earthdata.nasa.gov/wmts-webmerc/VIIRS_NOAA20_CorrectedReflectance_BandsM11-I2-I1/' +
            'default/{time}/GoogleMapsCompatible_Level9/{z}/{y}/{x}.jpg')
    src2 = (r'https://gibs.earthdata.nasa.gov/wmts-webmerc/VIIRS_NOAA20_CorrectedReflectance_TrueColor/' +
            'default/{time}/GoogleMapsCompatible_Level9/{z}/{y}/{x}.jpg')
    src3 = (r'https://gibs.earthdata.nasa.gov/wmts-webmerc/VIIRS_NOAA20_CorrectedReflectance_BandsM3-I3-M11/' +
            'default/{time}/GoogleMapsCompatible_Level9/{z}/{y}/{x}.jpg')
    fire_map = folium.Map(tiles='OpenStreetMap', min_zoom=5,
                          no_wrap=True, control_scale=(True), crs='EPSG3857')
    active_fire = folium.FeatureGroup('Thermal anomalies detected')
    if not in_ukraine.empty:
        for r in in_ukraine.iterrows():
            location = (r[1]['geometry'].coords[0])
            folium.Marker(location=location, tooltip=(
                r[1].to_string())).add_to(active_fire)
        if address_list:
            for address in address_list:
                if address is not None:
                    location = (address.raw['lat'], address.raw['lon'])
                    folium.CircleMarker(location=location, tooltip=(
                        str(address.raw['display_name'])),
                        radius=10).add_to(active_fire)
    fire_map.fit_bounds([(44.184598, 22.137059), (52.3797464, 40.2275801)])
    ukraine_borders = gpd.read_file('data.zip')
    folium.GeoJson(
        data=ukraine_borders['geometry'], name='Borders of Ukraine').add_to(fire_map)
    folium.raster_layers.TileLayer(
        tiles=src1,
        min_zoom=5,
        subdomains='abc',
        name='VIIRS CorrectedReflectance_BandsM11-I2-I1',
        attr='NASA VIIRS',
        overlay=True,
        layer='VIIRS_NOAA20_CorrectedReflectance_BandsM11-I2-I1',
        tileMatrixSet='GoogleMapsCompatible_Level9',
        time=str(dt.date.fromisoformat(date) - dt.timedelta(days=1)),
        tileSize=256,
    ).add_to(fire_map)
    folium.raster_layers.TileLayer(
        tiles=src2,
        min_zoom=5,
        subdomains='abc',
        name='VIIRS CorrectedReflectance TrueColor',
        attr='NASA VIIRS',
        overlay=True,
        layer='VIIRS_NOAA20_CorrectedReflectance_TrueColor',
        tileMatrixSet='GoogleMapsCompatible_Level9',
        time=str(dt.date.fromisoformat(date) - dt.timedelta(days=1)),
        tileSize=256,
    ).add_to(fire_map)
    folium.raster_layers.TileLayer(
        tiles=src3,
        min_zoom=5,
        subdomains='abc',
        name='VIIRS CorrectedReflectance BandsM3-I3-M11',
        attr='NASA VIIRS',
        overlay=True,
        layer='VIIRS_NOAA20_CorrectedReflectance_BandsM3-I3-M11',
        tileMatrixSet='GoogleMapsCompatible_Level9',
        time=str(dt.date.fromisoformat(date) - dt.timedelta(days=1)),
        tileSize=256,
    ).add_to(fire_map)
    active_fire.add_to(fire_map)
    if args.username:
        sar_area = download_sars_data(args, date)
        sar_area.add_to(fire_map)
    folium.map.LayerControl().add_to(fire_map)
    #fire_map = playground(fire_map)
    return fire_map


def filter_coords(data_frame, bounding):
    data_frame = data_frame.astype({'CONFIDENCE': 'string'})
    in_ukraine = data_frame[data_frame['LATITUDE'].between(bounding[0], bounding[1])
                            & data_frame['LONGITUDE'].between(bounding[2],
                                                              bounding[3])
                            & (data_frame['CONFIDENCE'].str.contains('nominal')
                               | data_frame['CONFIDENCE'].str.contains('high'))]
    return in_ukraine


def load_inputfile(args):
    if args.inputfile:
        local_file = args.inputfile
        date = re.search(r'\d{4}-\d{2}-\d{2}', args.inputfile).group(0)
    else:
        date = str(dt.date.today())
        local_file = 'local_VIIRS_data-' + date + '.zip'
    if not os.path.exists(local_file):
        try:
            remote_url = r'https://firms.modaps.eosdis.nasa.gov/data/active_fire/noaa-20-viirs-c2/shapes/zips/J1_VIIRS_C2_Europe_24h.zip'
            data = requests.get(remote_url, allow_redirects=True)
            try:
                with open(local_file, 'wb') as file:
                    file.write(data.content)
            except OSError as e:
                raise SystemExit(e)
        except requests.exceptions.RequestException as e:
            raise SystemExit(e)
    html_file = local_file.split('.')[0] + '.html'
    try:
        geo_data_frame = gpd.read_file(local_file)
    except Exception as e:
        SystemExit(e)
    geo_data_frame['geometry'] = geo_data_frame['geometry'].apply(
        lambda row: Point(row.y, row.x))
    return local_file, html_file, geo_data_frame, date


def download_sars_data(args, date):
    if not args.password:
        args.password = getpass.getpass('Password: ')
    try:
        api = SentinelAPI(args.username, args.password)
    except Exception as e:
        SystemExit(e)
    bounding_geojson = {
        "type": "FeatureCollection",
        "features": [
            {
                "type": "Feature",
                "properties": {},
                "geometry": {
                    "type": "Polygon",
                    "coordinates": [
                        [
                            [
                                22.137059,
                                44.184598
                            ],
                            [
                                40.2275801,
                                44.184598
                            ],
                            [
                                40.2275801,
                                52.3797464
                            ],
                            [
                                22.137059,
                                52.3797464
                            ],
                            [
                                22.137059,
                                44.184598
                            ]
                        ]
                    ]
                }
            }
        ]
    }
    footprint = geojson_to_wkt(bounding_geojson)
    products = api.query(
        area=footprint,
        date=(dt.date.fromisoformat(date) - dt.timedelta(days=1),
              dt.date.fromisoformat(date)),
        platformname='Sentinel-1',
        producttype='GRD')
    geo_data_frame = api.to_geodataframe(products)
    geo_data_frame['beginposition'] = geo_data_frame['beginposition'].astype(
        str)
    geo_data_frame['endposition'] = geo_data_frame['endposition'].astype(str)
    geo_data_frame['ingestiondate'] = geo_data_frame['ingestiondate'].astype(
        str)
    return folium.GeoJson(geo_data_frame.to_json(), name=('Areas covered by SAR'))


def playground(map):
    figure = map.get_root()
    element = Element('alert("hi there");')
    figure.script.add_child(element)
    return map


def main(args):
    bounding = [44.184598, 52.3797464, 22.137059, 40.2275801]
    local_file, html_file, geo_data_frame, date = load_inputfile(args)
    in_ukraine = filter_coords(geo_data_frame, bounding)
    if not args.noreversegeolocation:
        address_list = get_address(in_ukraine, date)
    else:
        address_list = []
    fire_map = render_map(args, in_ukraine, address_list, date)
    fire_map.save(html_file)
    webbrowser.open_new(html_file)
    SystemExit(0)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=('Downloads the actual active fire csv from ' +
                                                  'https://firms.modaps.eosdis.nasa.gov/data/active_fire/noaa-20-viirs-c2/csv/J1_VIIRS_C2_Europe_24h.csv ' +
                                                  'and displays it on a map. ' +
                                                  'Description of data displayed: ' +
                                                  'https://www.earthdata.nasa.gov/learn/find-data/near-real-time/firms/vnp14imgtdlnrt#ed-viirs-375m-attributes'))
    parser.add_argument(
        '-i', '--inputfile', type=check_file_format,
        help='Specify the cvs file containing the satellite data to be displayed. INPUTFILE must match local_VIIRS_data-YYYY-MM-DD.csv')
    parser.add_argument('-nr', '--noreversegeolocation', action='store_true',
                        help='Disable reverse geolocation')
    parser.add_argument('-u', '--username', type=str,
                        help='Username for https://scihub.copernicus.eu/')
    parser.add_argument('-p', '--password', type=str,
                        help='Password for https://scihub.copernicus.eu/')
    args = parser.parse_args()
    main(args)