Refactor project with justfile

.github/workflows/build.yml

@@ -13,39 +13,32 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    env:
-      BGPTOOLS_VERSION: 0.2.2
     steps:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@stable
       - uses: cargo-bins/cargo-binstall@main
       - run: sudo apt-get install -y tree jq axel
+      - uses: extractions/setup-just@v3
       - uses: actions/cache@v3
         with:
           key: ${{ runner.os }}-build-dependencies
           path: |
             ~/.cargo/bin
-      - run: ./dependency.sh
+      - run: just dependency
       - uses: nick-fields/retry@v3
         with:
           timeout_minutes: 60
           max_attempts: 3
-          command: ./generate.sh
+          command: just
-      - run: ./stat.sh
+      - run: just guard
-      - run: ./guard.sh
       - name: Checkout ip-lists branch
         uses: actions/checkout@v3
         if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'
         with:
           ref: ip-lists
           path: ip-lists
-      - run: ./upload.sh
+      - run: just upload
         if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'
-      - name: Refresh CDN cache
+      - run: just refresh_jsdelivr ${{ github.repository }}
         if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'
-        run: |
-          cd ip-lists
-          for file in *; do
-            curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@ip-lists/${file}"
-          done
 

common.sh

@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-[[ $DEBUG == true ]] && set -x
-
-log_info(){
-	>&2 echo "INFO>" $@
-}
-
-get_asn(){
-	local CONF_FILE=$1
-	unset PATTERN
-	unset COUNTRY
-	unset EXCLUDE
-	source $CONF_FILE
-	EXCLUDE=${EXCLUDE:-"^$"}
-	grep -P "${COUNTRY}\$" asnames.txt |
-	grep -Pi "$PATTERN" |
-	grep -vPi "$EXCLUDE" |
-	awk '{gsub(/AS/, ""); print $1 }'
-}
-
-prepare_data_v4(){
-	rm -f rib.gz
-	bgpkit-broker latest -c rrc00 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib.gz
-	stat rib.gz
-	log_info "rib.gz ready for bgptools"
-}
-prepare_data_v6(){
-	rm -f rib6.bz2
-	bgpkit-broker latest -c route-views6 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib6.bz2
-	stat rib6.bz2
-	log_info "rib6.bz2 ready for bgptools"
-}
-prepare_data(){
-	curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt &
-	prepare_data_v4 &
-	prepare_data_v6 &
-	wait_exit
-}
-
-wait_exit(){
-	local oldstate=$(set +o)
-	set +e
-	local s=0
-	while [[ $s -ne 127 ]]; do
-		[[ $s -ne 0 ]] && exit $s
-		wait -n
-		s=$?
-	done
-	eval "$oldstate"
-	return 0
-}

dependency.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-bgptools --version | grep -F $BGPTOOLS_VERSION || \
-    cargo install --version $BGPTOOLS_VERSION bgptools
-
-bgpkit-parser --version || \
-    cargo binstall --secure --no-confirm bgpkit-parser@0.10.6
-
-bgpkit-broker --version || \
-    cargo binstall --secure --no-confirm bgpkit-broker@0.7.0
-
-bgptools --version
-bgpkit-parser --version
-bgpkit-broker --version

generate.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-source common.sh
-[[ $SKIP_DATA_PREPARATION != true ]] && prepare_data
-mkdir -p result
-for file in operator/*.conf; do
-	operator=${file%.*}
-	operator=${operator##*/}
-	log_info "generating IP list of $operator ..."
-	get_asn $file
-	get_asn $file | xargs bgptools --mrt-file rib.gz   | grep -Fv : | cat > result/${operator}.txt  &
-	get_asn $file | xargs bgptools --mrt-file rib6.bz2 | grep -v '^::/0$' | grep -F  : | cat > result/${operator}6.txt &
-done
-
-wait_exit

guard.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-[[ $(wc -l result/china.txt) < 3000 ]] && exit 1
-
-[[ $(wc -l result/china6.txt) < 1000 ]] && exit 2
-
-exit 0

justfile

@@ -0,0 +1,215 @@
+set unstable
+set script-interpreter := ['bash']
+bgptools_version := "0.2.2"
+
+default: prepare all stat
+
+[doc('Install or update bgp tooling dependencies')]
+[script]
+dependency:
+  set -euo pipefail
+
+  if ! bgptools --version 2>/dev/null | grep -F "{{bgptools_version}}" >/dev/null; then
+    cargo install --version "{{bgptools_version}}" bgptools
+  fi
+
+  if ! bgpkit-broker --version >/dev/null 2>&1; then
+    cargo binstall --secure --no-confirm bgpkit-broker@0.7.0
+  fi
+
+  bgptools --version
+  bgpkit-broker --version
+
+[doc('Download and normalize latest autnums list')]
+[script]
+prepare_autnums:
+  set -euo pipefail
+
+  axel -q -o autnums.html https://bgp.potaroo.net/cidr/autnums.html
+  awk -F'[<>]' '{print $3,$5}' autnums.html | grep '^AS' > asnames.txt
+  rm -f autnums.html
+  echo "INFO> asnames.txt updated ($(wc -l < asnames.txt) entries)" >&2
+
+[doc('Download the latest IPv4 RIB snapshot')]
+[script]
+prepare_rib_v4:
+  set -euo pipefail
+
+  url="$(bgpkit-broker latest -c rrc00 --json \
+    | jq -r '.[] | select(.data_type | contains("rib")) | .url' \
+    | head -n 1)"
+
+  if [[ -z "${url}" ]]; then
+    echo "Unable to determine IPv4 RIB download url" >&2
+    exit 1
+  fi
+
+  rm -f rib.gz
+  axel -q -o rib.gz "${url}"
+  stat rib.gz
+  echo "INFO> rib.gz ready for bgptools" >&2
+
+[doc('Download the latest IPv6 RIB snapshot')]
+[script]
+prepare_rib_v6:
+  set -euo pipefail
+
+  url="$(bgpkit-broker latest -c route-views6 --json \
+    | jq -r '.[] | select(.data_type | contains("rib")) | .url' \
+    | head -n 1)"
+
+  if [[ -z "${url}" ]]; then
+    echo "Unable to determine IPv6 RIB download url" >&2
+    exit 1
+  fi
+
+  rm -f rib6.bz2
+  axel -q -o rib6.bz2 "${url}"
+  stat rib6.bz2
+  echo "INFO> rib6.bz2 ready for bgptools" >&2
+
+[doc('Prepare data for generation')]
+[parallel]
+prepare: prepare_autnums prepare_rib_v4 prepare_rib_v6
+
+[doc('Print ASN list for OPERATOR based on operator/*.conf')]
+[script]
+get_asn operator:
+  set -euo pipefail
+
+  config="operator/{{operator}}.conf"
+  if [[ ! -f "${config}" ]]; then
+    echo "Unknown operator: {{operator}}" >&2
+    exit 1
+  fi
+
+  if [[ ! -s asnames.txt ]]; then
+    echo "Missing asnames.txt. Run 'just prepare_autnums' first." >&2
+    exit 1
+  fi
+
+  # shellcheck disable=SC1090
+  source "${config}"
+  : "${COUNTRY:?COUNTRY must be set in ${config}}"
+  EXCLUDE="${EXCLUDE:-^$}"
+  PATTERN="${PATTERN:-}"
+
+  grep -P "${COUNTRY}\$" asnames.txt \
+    | grep -Pi "${PATTERN}" \
+    | grep -vPi "${EXCLUDE}" \
+    | awk '{gsub(/AS/, ""); print $1 }'
+
+[doc('Generate IP lists for a single operator')]
+[parallel]
+gen operator: (gen4 operator) (gen6 operator)
+
+[script]
+gen4 operator:
+  set -euo pipefail
+
+  mkdir -p result
+
+  echo "INFO> generating IPv4 prefixes for {{operator}}" >&2
+  just get_asn "{{operator}}" \
+    | tee >(awk 'END { if (NR == 0) exit 1 }') \
+    | xargs bgptools --mrt-file rib.gz \
+    | grep -Fv ':' \
+    > "result/{{operator}}.txt"
+  echo "INFO> {{operator}}.txt generated ($(wc -l < result/{{operator}}.txt) entries)" >&2
+
+[script]
+gen6 operator:
+  set -euo pipefail
+
+  mkdir -p result
+
+  echo "INFO> generating IPv6 prefixes for {{operator}}" >&2
+  just get_asn "{{operator}}" \
+    | tee >(awk 'END { if (NR == 0) exit 1 }') \
+    | xargs bgptools --mrt-file rib6.bz2 \
+    | grep -v '^::/0$' \
+    | grep -F ':' \
+    > "result/{{operator}}6.txt" || true  # ignore empty output, since drpeng has no IPv6 prefixes
+  echo "INFO> {{operator}}6.txt generated ($(wc -l < result/{{operator}}6.txt) entries)" >&2
+
+[doc('Generate IP lists for all operators sequentially')]
+[parallel]
+all: (gen "china") (gen "cernet") (gen "chinanet") (gen "cmcc") (gen "unicom") (gen "cstnet") (gen "drpeng") (gen "googlecn")
+
+[script]
+guard:
+  set -euo pipefail
+
+  if [[ $(wc -l < result/china.txt) -lt 3000 ]]; then
+    echo "china.txt too small" >&2
+    exit 1
+  fi
+
+  if [[ $(wc -l < result/china6.txt) -lt 1000 ]]; then
+    echo "china6.txt too small" >&2
+    exit 2
+  fi
+
+  echo "INFO> guard checks passed" >&2
+
+[doc('Summarize total IPv4/IPv6 address space per operator')]
+[script]
+stat:
+  set -euo pipefail
+  cd result
+
+  for file in *.txt; do
+    name="${file%.*}"
+    echo "${name}"
+    if [[ "${file}" == *6.txt ]]; then
+      base=48
+    else
+      base=32
+    fi
+
+    sum=0
+    while IFS=/ read -r _ mask; do
+      if [[ -z "${mask}" ]]; then
+        continue
+      fi
+      if (( mask <= base )); then
+        ((s=base-mask))
+        ((sum+=1<<s))
+      fi
+    done < "${file}"
+    echo "${sum}"
+    echo
+  done | tee stat
+
+[doc('Publish generated results into the ip-lists branch')]
+[script]
+upload: guard
+  set -euo pipefail
+  rm -f ip-lists/*.txt
+  mv result/* ip-lists
+  cd ip-lists
+  tree -H . -P "*.txt|stat" -T "China Operator IP - prebuild results" > index.html
+  git config user.name "GitHub Actions"
+  git config user.email noreply@github.com
+  git add .
+  git commit -m "update $(date +%Y-%m-%d)"
+  git push -q
+
+[doc('Refresh CDN cache for all files in ip-lists directory')]
+[script]
+refresh_jsdelivr repository:
+  set -euo pipefail
+
+  if [[ ! -d ip-lists ]]; then
+    echo "ip-lists directory not found" >&2
+    exit 1
+  fi
+
+  cd ip-lists
+  for file in *; do
+    if [[ -f "${file}" ]]; then
+      echo "INFO> purging CDN cache for ${file}" >&2
+      curl -i "https://purge.jsdelivr.net/gh/{{repository}}@ip-lists/${file}"
+    fi
+  done
+

stat.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-source common.sh
-cd result
-for file in *.txt; do
-	echo ${file%.*}
-	if [[ $file == *6.txt ]]; then
-		#statistics IPv6 /48 blocks
-		base=48
-	else
-		base=32
-	fi
-	cat $file |
-	awk -F\/ '{print $2}' |
-	(
-		sum=0
-		while read n; do
-			if [[ -n $n ]] && [[ $n -le $base ]]; then
-				((s=base-n))
-				((sum+=1<<s))
-			fi
-		done
-		echo $sum
-	)
-	echo
-done | tee stat

upload.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-rm ip-lists/*.txt
-mv result/* ip-lists
-cd ip-lists
-tree -H . -P "*.txt|stat" -T "China Operator IP - prebuild results" > index.html
-git config user.name "GitHub Actions"
-git config user.email noreply@github.com
-git add .
-git commit -m "update $(date +%Y-%m-%d)"
-git push -q