Files
tool__scripts/auto-curler/auto-curler.sh
2026-07-09 01:05:56 +02:00

47 lines
926 B
Bash
Executable File

#!/bin/bash
if [ $# -eq 0 ]; then
echo "Usage: $0 file_with_urls.txt"
exit 1
fi
URL_FILE="$1"
OUTPUT_DIR="output"
mkdir -p "$OUTPUT_DIR"
while IFS= read -r url; do
if [ -z "$url" ]; then
continue
fi
domain=$(echo "$url" | awk -F[/:] '{print $4}')
filename=$(basename "$url")
sub_dir="$OUTPUT_DIR/$domain"
full_path="${sub_dir}/${filename}"
mkdir -p "$sub_dir"
if [ ! -f "$full_path" ]; then
echo "Downloading $url..."
echo "Subdir $sub_dir"
curl -o "$full_path" "$url"
last_modified=$(curl -sI $url | grep -i Last-Modified | awk '{print substr($0, index($0,$2))}')
echo "Modified: $last_modified"
touch -d "$last_modified" "$full_path"
echo "---"
else
echo "Skipping $url; file already exists."
fi
sleep 2
done < "$URL_FILE"
echo "Download complete."