scripts/bash/scrapeMoonrank.sh

#!/bin/bash

# This script will scrape moonrank.app and save a json file of the
#  rarities for all NFTs of a collection (must be fully minted).

########## Configurable Variables ##########
# Where to save the json files             #
# File location: (folder)                  #
  directory="~"                            #
                                           #
# How long to wait for a response          #
# Curl timeout: (seconds)                  #
  ctimeout=5                               #
                                           #
# How long to sleep between web queries    #
# Delay: (seconds)                         #
  delay=2                                  #
                                           #
# How many checks to do between sleeps     #
# Count: (number)                          #
  sequential=2                             #
                                           #
############################################


# Colors are fun
blk=$(tput setaf 0);red=$(tput setaf 1);grn=$(tput setaf 2)
ylw=$(tput setaf 3);blu=$(tput setaf 4);cyn=$(tput setaf 6)
wht=$(tput setaf 7);und=$(tput smul);bld=$(tput bold)
# Uncoloring is important
rst=$(tput sgr0)

json=""
max=0
cur=0
moonrank="https://moonrank.app/collection/"

trap ctrl_c INT
function ctrl_c() {
 tput cnorm
 echo "${rst}${red}CTRL+C Detected, exiting.${rst}"
 exit
}

function timedout() {
 echo -e "\n${red}No response received for ${bld}${1}${rst}${red}."
 echo -e "Please ensure you are able to access the internet. If this problem"
 echo -e "persists perhaps moonrank.app is unavailable. Consult your friends.${rst}\n"
 exit 1
}

function verifyurl() {
 status=`curl -I -m ${ctimeout} "$1" 2>/dev/null | head -n1 | cut -d' ' -f2`
 [ "$status" == "" ] && timedout "${1}"
 [ $status -eq 200 ] && return 0 || return 1
}

function getcollectioninfo() {
 local moonrankdata=`curl -m ${ctimeout} ${moonrank}${collection} 2>/dev/null`
 [ "${moonrankdata}" == "" ] && timedout "${1}"
 name=`echo "${moonrankdata}" | grep "font-medium text-white tracking-tight hyphens-auto" -a1 | tail -n1`
 count=`echo "${moonrankdata}" | egrep -o "[0-9,]+ pieces" | cut -d' ' -f1`
 verifyurl "${moonrank}${collection}/0"
 zeroindexed=`[ $? -eq 0 ] && echo yes || echo no`
}

# Sort through the HTML and save the rarity
function getrarity() {
 local moonrankdata=`curl -m ${ctimeout} ${moonrank}${collection}/${cur} 2>/dev/null`
 [ "${moonrankdata}" == "" ] && timedout "${1}"
 rarity=`echo "${moonrankdata}" | egrep -o "text=\"rank\">\d+" | egrep -o "\d+"`
 nftname=`echo "${moonrankdata}" | egrep -o "aria\-current=\"page\">[a-zA-Z ]+#[0-9]+" | cut -d">" -f2`
}

# $1=key $2=value
function appendjson() {
 [ "${#json}" -eq 0 ] && json="{\"$1\":$2" || json="${json}, \"$1\":$2"
}

function pad() {
 [ $1 -ne 0 ] && seq -s " " $1 | tr -d "[:digit:]"
}

function isodd(){
 local num=$1
 [ $num -eq 0 ] && return 0
 [ $num -lt 0 ] && let num="num*-1"
 local a=0;local b=0
 let a="num/2"
 let b="(num-1)/2"
 [ $a -eq $b ] && return 1
 return 0
}

function tableheader() {
 [ $didheader -eq 2 ] && return 1
 didheader=0
 local namelen=${#nftname}
 isodd $namelen
 local odd=$?
 if [[ $namelen -gt 7 ]]; then
  let prepost="namelen-7"
  let prepost="prepost/2"
 elif [[ $namelen -lt 7 ]]; then
  let prepost="7-namelen"
  let prepost="prepost/2"
 else
  prepost=0
 fi

 for i in `seq 2`; do
  [ $didheader -eq 0 ] && echo -n " "
  pad $prepost
  [ $prepost -ne 0 ] && echo -n " "
  echo -n "NFT ID"
  [ $odd -eq 0 ] && let prepost="prepost+1"
  pad $prepost
  [ $prepost -eq 0 ] && echo -en "\t"
  echo -en "\tNFT Rarity Rank"
  [ $didheader -eq 0 ] && echo -en "\t\t"
  [ $didheader -eq 1 ] && echo
 let didheader="didheader+1"
 done
 tablecolumn=1
}

function tablerow() {
 if [[ $tablecolumn -eq 1 ]]; then
  echo -en " ${nftname}\t     ${rarity}"
  tablecolumn=2
 else
  echo -e "\t\t ${nftname}\t      ${rarity}"
  tablecolumn=1
 fi
}

####### START

yn=n
while [[ $yn == n ]]; do
 echo -en "\nPlease enter the moonrank collection ID: ${cyn}"
 read collection
 echo ${rst}
 collection=`echo $collection | tr [:blank:] _`
 echo -en "You entered ${bld}${collection}${rst}. Is this correct? [${grn}Y${rst}/${red}n${rst}]: ${cyn}"
 read yn
 echo -en "${rst}"
done
 
verifyurl ${moonrank}${collection}
[ $? -eq 1 ] && echo -e "\n${red}${bld}${collection}${rst}${red} could not be found. Please ensure the name is correct and try again.${rst}\n" && exit 1
getcollectioninfo

echo -e "\nFound the ${bld}${name}${rst} collection of ${bld}${count}${rst} NFTs."
max=`echo ${count} | tr -d ,`
[ $zeroindexed == "yes" ] && let max="max-1" || cur=1

echo -e "\n"

didheader=0
thisseq=0
getrarity
tableheader
while [[ $cur -le $max ]]; do
 getrarity
 tablerow
 appendjson "${nftname}" ${rarity}
 let cur="cur+1"
 let thisseq="thisseq+1"
 [ $thisseq -eq $sequential ] && thisseq=0 && sleep $delay
done

json="${json}}"

echo "$json" > ${directory}/${collection}-rarity.json
echo -e "\nThe .json file for the ${name} NFT collection has been saved to ${directory}/${collection}-rarity.json\n"