#!/bin/bash
# 2025-02-14
# stephanie.leroux@datlas.fr
# Example how to download all files from the opendap server
# NOTE: this script works with a mac os. On a lminux machine you might need to adjust the syntax of the sed command by removing '' after -i option

# === 0. SET params  ===
YYYY=2024  # Set the year as a variable

# Base URL
base_url="https://ige-meom-opendap.univ-grenoble-alpes.fr/thredds/catalog/meomopendap/extract/MEOM/NeXtSIM-F"


echo "==== 1. Prepare lists of files to be downloaded (month by month) ===="

# Loop over months 01 to 12
for month in $(seq -w 1 12); do
    echo "Processing month: $month"

    # Construct catalog URL
    catalog_url="${base_url}/${YYYY}/${month}/catalog.xml"

    # Download and extract file names
    wget -q -O - "$catalog_url" | grep -o 'dataset name="[^"]*\.nc"' | cut -d '"' -f 2 > "file_list_${month}.txt"

    # Check if file list is empty
    if [ ! -s "file_list_${month}.txt" ]; then
        echo "No files found for month $month, skipping..."
        continue
    fi

    # Prepend the correct fileServer URL
    sed -i '' "s|^|https://ige-meom-opendap.univ-grenoble-alpes.fr/thredds/fileServer/meomopendap/extract/MEOM/NeXtSIM-F/${YYYY}/${month}/|" "file_list_${month}.txt"

    echo "File list for $month created: file_list_${month}.txt"
done

echo "All file lists created!"

echo "==== 2. Now downloading all files into ${YYYY}/MM/ directories (4 downloads in parallel) ===="

# Loop through months and download files into YYYY/MM directories
for month in $(seq -w 1 12); do
    if [ -s "file_list_${month}.txt" ]; then
        # Create directory for the month
        mkdir -p "${YYYY}/${month}"

        # Download files into the correct directory
        cat "file_list_${month}.txt" | xargs -n 1 -P 4 -I {} wget -P "${YYYY}/${month}" {}
    fi
done

echo "==== Download complete! Files saved in ${YYYY}/MM/ directories ===="

