Skip to content

Commit 617923a

Browse files
igorpecovnikigorpecovnik
authored
Enrich JSON generation by pulling data for Bigin (#104)
* Improve bigin pull * Update index gen v2 * Add pipelines * Drop company description * Add vendors generation * Fix
1 parent 8b88352 commit 617923a

2 files changed

Lines changed: 248 additions & 14 deletions

File tree

‎.github/workflows/update-download-index.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
name: Update download JSON index v2
22

33
on:
4-
push:
54
repository_dispatch:
65
types: ["Webindex update"]
76
workflow_dispatch:
@@ -59,10 +58,17 @@ jobs:
5958
- name: Install dependencies
6059
run: |
6160
sudo apt-get update
62-
sudo apt-get install -y jq jc rsync
61+
sudo apt-get install -y jq jc rsync curl
6362
64-
- name: Generate armbian-images.json
63+
- name: Generate armbian-images.json (with Bigin company enrichment)
6564
working-directory: runner
65+
env:
66+
# Zoho OAuth credentials for Bigin enrichment
67+
ZOHO_CLIENT_ID: ${{ secrets.ZOHO_CLIENT_ID }}
68+
ZOHO_CLIENT_SECRET: ${{ secrets.ZOHO_CLIENT_SECRET }}
69+
ZOHO_REFRESH_TOKEN: ${{ secrets.ZOHO_REFRESH_TOKEN }}
70+
# You can also disable enrichment if needed:
71+
# BIGIN_ENABLE: "false"
6672
run: |
6773
set -euo pipefail
6874
OUT="armbian-images.json" \
@@ -76,7 +82,6 @@ jobs:
7682
7783
- name: Commit changes if any
7884
run: |
79-
8085
set -euo pipefail
8186
cd armbian.github.io
8287

‎scripts/generate-armbian-images-json.sh‎

Lines changed: 239 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,35 @@ OS_DIR="${OS_DIR:-./os}"
1010
BOARD_DIR="${BOARD_DIR:-./build/config/boards}"
1111
OUT="${OUT:-armbian-images.json}"
1212

13+
# -----------------------------------------------------------------------------
14+
# Zoho Bigin configuration (optional enrichment)
15+
# -----------------------------------------------------------------------------
16+
BIGIN_ENABLE="${BIGIN_ENABLE:-true}"
17+
BIGIN_API_BASE="${BIGIN_API_BASE:-https://www.zohoapis.eu/bigin/v2}"
18+
ZOHO_OAUTH_TOKEN_URL="${ZOHO_OAUTH_TOKEN_URL:-https://accounts.zoho.eu/oauth/v2/token}"
19+
20+
# Accounts: custom field that contains company slug (must match board_vendor)
21+
BIGIN_COMPANY_SLUG_FIELD="${BIGIN_COMPANY_SLUG_FIELD:-Company_slug}"
22+
BIGIN_ACCOUNT_FIELDS="Account_Name,Website,${BIGIN_COMPANY_SLUG_FIELD}"
23+
24+
# Pipelines (confirmed keys): Boards, Closing_Date, Stage
25+
BIGIN_PLATINUM_MODULE="${BIGIN_PLATINUM_MODULE:-Pipelines}"
26+
BIGIN_PLATINUM_BOARDS_FIELD="${BIGIN_PLATINUM_BOARDS_FIELD:-Boards}"
27+
BIGIN_PLATINUM_UNTIL_FIELD="${BIGIN_PLATINUM_UNTIL_FIELD:-Closing_Date}"
28+
BIGIN_PLATINUM_STATUS_FIELD="${BIGIN_PLATINUM_STATUS_FIELD:-Stage}"
29+
BIGIN_PLATINUM_FIELDS="${BIGIN_PLATINUM_STATUS_FIELD},${BIGIN_PLATINUM_UNTIL_FIELD},${BIGIN_PLATINUM_BOARDS_FIELD}"
30+
1331
# -----------------------------------------------------------------------------
1432
# Requirements
1533
# -----------------------------------------------------------------------------
1634
need() { command -v "$1" >/dev/null || { echo "ERROR: missing '$1'" >&2; exit 1; }; }
17-
need rsync gh jq jc find grep sed cut awk sort mktemp
35+
need rsync gh jq jc find grep sed cut awk sort mktemp curl date
1836

1937
[[ -f "${OS_DIR}/exposed.map" ]] || { echo "ERROR: ${OS_DIR}/exposed.map not found" >&2; exit 1; }
2038
[[ -d "${BOARD_DIR}" ]] || { echo "ERROR: board directory not found: ${BOARD_DIR}" >&2; exit 1; }
2139

40+
TODAY_UTC="$(date -u +%F)"
41+
2242
# -----------------------------------------------------------------------------
2343
# Extract variable from board config
2444
# -----------------------------------------------------------------------------
@@ -39,8 +59,8 @@ extract_cfg_var() {
3959
# -----------------------------------------------------------------------------
4060
# Load board metadata + track incomplete metadata (file-based, set -u safe)
4161
# -----------------------------------------------------------------------------
42-
declare -A BOARD_NAME_MAP
43-
declare -A BOARD_VENDOR_MAP
62+
declare -A BOARD_NAME_MAP=()
63+
declare -A BOARD_VENDOR_MAP=()
4464

4565
MISSING_META_FILE="$(mktemp)"
4666
trap 'rm -f "$MISSING_META_FILE"' EXIT
@@ -64,6 +84,194 @@ done < <(
6484
| sort
6585
)
6686

87+
# -----------------------------------------------------------------------------
88+
# Optional: Load company data from Bigin keyed by company_slug (matches board_vendor)
89+
# -----------------------------------------------------------------------------
90+
declare -A COMPANY_NAME_BY_SLUG=()
91+
declare -A COMPANY_WEBSITE_BY_SLUG=()
92+
93+
# Platinum support: store latest until-date per board_slug
94+
declare -A PLATINUM_UNTIL_BY_BOARD=()
95+
96+
get_zoho_access_token() {
97+
local client_id="${ZOHO_CLIENT_ID:-}"
98+
local client_secret="${ZOHO_CLIENT_SECRET:-}"
99+
local refresh_token="${ZOHO_REFRESH_TOKEN:-}"
100+
101+
if [[ -z "$client_id" || -z "$client_secret" || -z "$refresh_token" ]]; then
102+
echo ""
103+
return 0
104+
fi
105+
106+
curl -sH "Content-type: multipart/form-data" \
107+
-F refresh_token="$refresh_token" \
108+
-F client_id="$client_id" \
109+
-F client_secret="$client_secret" \
110+
-F grant_type=refresh_token \
111+
-X POST "$ZOHO_OAUTH_TOKEN_URL" \
112+
| jq -r '.access_token // empty'
113+
}
114+
115+
# Keep the latest ISO date/timestamp string lexicographically
116+
max_date() {
117+
local a="$1" b="$2"
118+
[[ -z "$a" ]] && { echo "$b"; return; }
119+
[[ -z "$b" ]] && { echo "$a"; return; }
120+
[[ "$a" < "$b" ]] && echo "$b" || echo "$a"
121+
}
122+
123+
# Convert "2025-06-25" or "2025-06-25T..." -> "2025-06-25"
124+
date_only() {
125+
local s="$1"
126+
s="${s%%T*}"
127+
echo "$s"
128+
}
129+
130+
load_bigin_companies() {
131+
local token="$1"
132+
[[ -n "$token" ]] || return 0
133+
134+
echo "▶ Fetching Bigin company data…" >&2
135+
echo " - fields: ${BIGIN_ACCOUNT_FIELDS}" >&2
136+
echo " - company slug field: ${BIGIN_COMPANY_SLUG_FIELD}" >&2
137+
echo " - join key: board_vendor == company_slug" >&2
138+
139+
local page=1 per_page=200 more="true"
140+
local loaded=0
141+
142+
while [[ "$more" == "true" ]]; do
143+
local resp="/tmp/bigin-accounts-${page}.json"
144+
145+
curl -s \
146+
-H "Authorization: Zoho-oauthtoken ${token}" \
147+
"${BIGIN_API_BASE}/Accounts?fields=${BIGIN_ACCOUNT_FIELDS}&per_page=${per_page}&page=${page}" \
148+
> "$resp"
149+
150+
if ! jq -e '.data' "$resp" >/dev/null 2>&1; then
151+
echo "WARNING: Bigin Accounts response missing .data (page=${page}); skipping company enrichment." >&2
152+
jq '.' "$resp" >&2 || true
153+
return 0
154+
fi
155+
156+
while IFS=$'\t' read -r slug name website desc; do
157+
slug="${slug,,}"
158+
[[ -z "$slug" ]] && continue
159+
COMPANY_NAME_BY_SLUG["$slug"]="$name"
160+
COMPANY_WEBSITE_BY_SLUG["$slug"]="$website"
161+
((loaded++)) || true
162+
done < <(
163+
jq -r --arg f "$BIGIN_COMPANY_SLUG_FIELD" '
164+
(.data // [])
165+
| map({
166+
slug: (.[ $f ] // "" | tostring),
167+
name: (.Account_Name // "" | tostring),
168+
website: (.Website // "" | tostring),
169+
})
170+
| .[]
171+
| select(.slug != "")
172+
| [.slug,.name,.website,.desc] | @tsv
173+
' "$resp"
174+
)
175+
176+
more="$(jq -r '.info.more_records // false' "$resp")"
177+
page=$((page + 1))
178+
[[ "$page" -le 50 ]] || { echo "WARNING: Bigin Accounts pagination safety cap hit; stopping." >&2; break; }
179+
done
180+
181+
echo " - Bigin company slugs loaded: ${#COMPANY_NAME_BY_SLUG[@]} (rows processed: ${loaded})" >&2
182+
}
183+
184+
load_bigin_platinum_support() {
185+
local token="$1"
186+
[[ -n "$token" ]] || return 0
187+
188+
echo "▶ Fetching Bigin platinum support from ${BIGIN_PLATINUM_MODULE}…" >&2
189+
echo " - fields: ${BIGIN_PLATINUM_FIELDS}" >&2
190+
echo " - rule: map Boards tokens -> board_slug; ignore Stage=Cancelled/Canceled; latest Closing_Date wins" >&2
191+
192+
local per_page=200
193+
local page_token=""
194+
local pages=0
195+
local rows=0
196+
local nonnull=0
197+
198+
while :; do
199+
pages=$((pages + 1))
200+
[[ "$pages" -le 200 ]] || { echo "WARNING: pagination safety cap hit; stopping." >&2; break; }
201+
202+
local resp="/tmp/bigin-platinum-${pages}.json"
203+
local url="${BIGIN_API_BASE}/${BIGIN_PLATINUM_MODULE}?fields=${BIGIN_PLATINUM_FIELDS}&per_page=${per_page}"
204+
[[ -n "$page_token" ]] && url="${url}&page_token=${page_token}"
205+
206+
curl -s -H "Authorization: Zoho-oauthtoken ${token}" "$url" > "$resp"
207+
208+
if ! jq -e '.data' "$resp" >/dev/null 2>&1; then
209+
echo "WARNING: Bigin ${BIGIN_PLATINUM_MODULE} response missing .data; skipping platinum extraction." >&2
210+
jq '.' "$resp" >&2 || true
211+
return 0
212+
fi
213+
214+
# Count non-null Boards on this page (just for your debug summary)
215+
local page_nonnull
216+
page_nonnull="$(jq -r --arg bf "$BIGIN_PLATINUM_BOARDS_FIELD" '
217+
[(.data // [])[] | .[$bf] // empty | tostring | select(. != "" and . != "null")] | length
218+
' "$resp" 2>/dev/null || echo 0)"
219+
nonnull=$((nonnull + page_nonnull))
220+
221+
# IMPORTANT: no pipe into while; use process substitution to keep map updates
222+
while IFS=$'\t' read -r b until; do
223+
b="${b,,}"
224+
b="$(sed -E 's/^[[:space:]]+|[[:space:]]+$//g' <<<"$b")"
225+
[[ -z "$b" ]] && continue
226+
227+
until="$(date_only "$until")"
228+
[[ -z "$until" || "$until" == "null" ]] && continue
229+
230+
local cur="${PLATINUM_UNTIL_BY_BOARD[$b]:-}"
231+
PLATINUM_UNTIL_BY_BOARD["$b"]="$(max_date "$cur" "$until")"
232+
rows=$((rows + 1))
233+
done < <(
234+
jq -r \
235+
--arg bf "$BIGIN_PLATINUM_BOARDS_FIELD" \
236+
--arg uf "$BIGIN_PLATINUM_UNTIL_FIELD" \
237+
--arg sf "$BIGIN_PLATINUM_STATUS_FIELD" '
238+
(.data // [])
239+
| map(select(((.[ $sf ] // "") | tostring | ascii_downcase) | IN("cancelled","canceled") | not))
240+
| .[]
241+
| (.[ $uf ] // "" | tostring) as $until
242+
| (.[ $bf ] // "" | tostring) as $boards
243+
| select($boards != "" and $boards != "null")
244+
| ($boards
245+
| gsub("[\r\n\t]"; " ")
246+
| gsub("[;]+"; ",")
247+
| split(",")
248+
| map(gsub("^\\s+|\\s+$"; ""))
249+
| map(select(length>0))
250+
)[]
251+
| [., $until] | @tsv
252+
' "$resp"
253+
)
254+
255+
page_token="$(jq -r '.info.next_page_token // empty' "$resp")"
256+
[[ -n "$page_token" ]] || break
257+
done
258+
259+
echo " - pages read: ${pages}" >&2
260+
echo " - records with non-empty Boards seen: ${nonnull}" >&2
261+
echo " - platinum boards mapped: ${#PLATINUM_UNTIL_BY_BOARD[@]} (rows processed: ${rows})" >&2
262+
}
263+
264+
265+
if [[ "${BIGIN_ENABLE}" == "true" ]]; then
266+
ZOHO_TOKEN="$(get_zoho_access_token || true)"
267+
if [[ -n "${ZOHO_TOKEN}" ]]; then
268+
load_bigin_companies "${ZOHO_TOKEN}" || true
269+
load_bigin_platinum_support "${ZOHO_TOKEN}" || true
270+
else
271+
echo "ℹ️ Bigin enrichment disabled (missing Zoho secrets or token could not be obtained)." >&2
272+
fi
273+
fi
274+
67275
# -----------------------------------------------------------------------------
68276
# Helpers
69277
# -----------------------------------------------------------------------------
@@ -92,6 +300,8 @@ get_download_repository() {
92300
awk -F/ '{print $5}' <<<"$url"
93301
elif [[ "$url" == https://dl.armbian.com/* ]]; then
94302
awk -F/ '{print $5}' <<<"$url"
303+
elif [[ "$url" == https://dl.armbian.com/* ]]; then
304+
awk -F/ '{print $5}' <<<"$url"
95305
else
96306
echo ""
97307
fi
@@ -102,18 +312,14 @@ get_download_repository() {
102312
# -----------------------------------------------------------------------------
103313
EXPOSED_MAP_FILE="${OS_DIR}/exposed.map"
104314

105-
# Return 0 if given candidate matches any exposed pattern
106315
is_promoted_candidate() {
107316
local candidate="$1"
108-
# grep -E with -f treats each line as an ERE
109317
grep -Eq -f "$EXPOSED_MAP_FILE" <<<"$candidate"
110318
}
111319

112320
is_promoted() {
113-
# args: image_name board_slug url
114321
local image_name="$1" board_slug="$2" url="$3"
115322

116-
# Candidates to match: filename, board/archive/filename, and relative URL paths
117323
local rel_dl="${url#https://dl.armbian.com/}"
118324
local rel_cache="${url#https://cache.armbian.com/artifacts/}"
119325
local rel_github="${url#https://github.com/armbian/}"
@@ -126,7 +332,6 @@ is_promoted() {
126332
"$rel_cache" \
127333
"$rel_github"
128334
do
129-
# If stripping didn't change it, it won't hurt; if it did, it's useful.
130335
[[ "$c" == "$url" ]] && continue
131336
if is_promoted_candidate "$c"; then
132337
return 0
@@ -209,7 +414,7 @@ cat "$tmpdir/a.txt" "$tmpdir/bcd.txt" >"$feed"
209414
# JSON generation
210415
# -----------------------------------------------------------------------------
211416
{
212-
echo '"board_slug"|"board_name"|"board_vendor"|"armbian_version"|"file_url"|"file_url_asc"|"file_url_sha"|"file_url_torrent"|"redi_url"|"redi_url_asc"|"redi_url_sha"|"redi_url_torrent"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"'
417+
echo '"board_slug"|"board_name"|"board_vendor"|"company_name"|"company_website"|"company_logo"|"platinum_support"|"platinum_support_until"|"platinum_support_expired"|"armbian_version"|"file_url"|"file_url_asc"|"file_url_sha"|"file_url_torrent"|"redi_url"|"redi_url_asc"|"redi_url_sha"|"redi_url_torrent"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"'
213418

214419
while IFS="|" read -r SIZE URL DATE; do
215420
IMAGE_SIZE="${SIZE//[.,]/}"
@@ -251,7 +456,31 @@ cat "$tmpdir/a.txt" "$tmpdir/bcd.txt" >"$feed"
251456
PROMOTED=true
252457
fi
253458

254-
echo "${BOARD_SLUG}|${BOARD_NAME_MAP[$BOARD_SLUG]:-}|${BOARD_VENDOR_MAP[$BOARD_SLUG]:-}|${VER}|${URL}|${ASC}|${SHA}|${TOR}|${REDI_URL}|${REDI_URL}.asc|${REDI_URL}.sha|${REDI_URL}.torrent|${DATE}|${IMAGE_SIZE}|${DISTRO}|${BRANCH}|${VARIANT}|${APP}|${PROMOTED}|${REPO}|${FILE_EXTENSION}"
459+
BOARD_VENDOR="${BOARD_VENDOR_MAP[$BOARD_SLUG]:-}"
460+
COMPANY_KEY="${BOARD_VENDOR,,}"
461+
462+
C_NAME="${COMPANY_NAME_BY_SLUG[$COMPANY_KEY]:-}"
463+
C_WEB="${COMPANY_WEBSITE_BY_SLUG[$COMPANY_KEY]:-}"
464+
465+
C_LOGO=""
466+
if [[ -n "$BOARD_VENDOR" ]]; then
467+
C_LOGO="https://cache.armbian.com/images/vendors/150/${BOARD_VENDOR}.png"
468+
fi
469+
470+
PLAT_UNTIL="${PLATINUM_UNTIL_BY_BOARD[$BOARD_SLUG]:-}"
471+
472+
PLAT="false"
473+
PLAT_EXPIRED="false"
474+
if [[ -n "$PLAT_UNTIL" ]]; then
475+
if [[ "$PLAT_UNTIL" < "$TODAY_UTC" ]]; then
476+
PLAT="false"
477+
PLAT_EXPIRED="true"
478+
else
479+
PLAT="true"
480+
PLAT_EXPIRED="false"
481+
fi
482+
fi
483+
echo "${BOARD_SLUG}|${BOARD_NAME_MAP[$BOARD_SLUG]:-}|${BOARD_VENDOR}|${C_NAME}|${C_WEB}|${C_LOGO}|${PLAT}|${PLAT_UNTIL}|${PLAT_EXPIRED}|${VER}|${URL}|${ASC}|${SHA}|${TOR}|${REDI_URL}|${REDI_URL}.asc|${REDI_URL}.sha|${REDI_URL}.torrent|${DATE}|${IMAGE_SIZE}|${DISTRO}|${BRANCH}|${VARIANT}|${APP}|${PROMOTED}|${REPO}|${FILE_EXTENSION}"
255484
done <"$feed"
256485

257486
} | jc --csv | jq '{assets:.}' >"$OUT"

0 commit comments

Comments
 (0)