Enhance Forgejo LFS integration by adding REPO_ACTOR and improving matcher binary resolution
Some checks failed
Publish FaceAI Container / publish (push) Failing after 1m26s
Some checks failed
Publish FaceAI Container / publish (push) Failing after 1m26s
This commit is contained in:
parent
be39662d10
commit
41823ad60a
1 changed files with 113 additions and 56 deletions
|
|
@ -26,6 +26,7 @@ jobs:
|
|||
DOCKER_HOST: ${{ vars.DOCKER_HOST != '' && vars.DOCKER_HOST || 'tcp://172.17.0.1:2375' }}
|
||||
REPO_SERVER_URL: ${{ forgejo.server_url }}
|
||||
REPO_NAME: ${{ forgejo.repository }}
|
||||
REPO_ACTOR: ${{ forgejo.actor }}
|
||||
REPO_AUTH_USER: ${{ secrets.FORGEJO_LFS_USERNAME != '' && secrets.FORGEJO_LFS_USERNAME || 'x-access-token' }}
|
||||
REPO_TOKEN: ${{ secrets.FORGEJO_LFS_TOKEN != '' && secrets.FORGEJO_LFS_TOKEN || forgejo.token }}
|
||||
|
||||
|
|
@ -48,87 +49,143 @@ jobs:
|
|||
if [ ! -f "faceai/package.json" ]; then echo "faceai/package.json is missing from the repository checkout"; exit 1; fi
|
||||
if [ ! -f "bin/Face_Recognition_Unix/face_matcher" ]; then echo "bin/Face_Recognition_Unix/face_matcher is missing from the repository checkout"; exit 1; fi
|
||||
|
||||
- name: Ensure Git LFS exists
|
||||
run: |
|
||||
set -eu
|
||||
if command -v git-lfs >/dev/null 2>&1; then
|
||||
git-lfs version
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
apt-get install -y git-lfs
|
||||
git-lfs version
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ARCH="$(uname -m)"
|
||||
case "${ARCH}" in
|
||||
x86_64) LFS_ARCH="amd64" ;;
|
||||
aarch64|arm64) LFS_ARCH="arm64" ;;
|
||||
*) echo "Unsupported architecture for git-lfs bootstrap: ${ARCH}"; exit 1 ;;
|
||||
esac
|
||||
|
||||
GIT_LFS_VERSION="3.6.1"
|
||||
curl -fsSL "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/git-lfs-linux-${LFS_ARCH}-v${GIT_LFS_VERSION}.tar.gz" -o git-lfs.tgz
|
||||
tar -xzf git-lfs.tgz
|
||||
mkdir -p "${HOME}/.local/bin"
|
||||
find . -type f -name git-lfs -exec cp {} "${HOME}/.local/bin/git-lfs" \;
|
||||
chmod +x "${HOME}/.local/bin/git-lfs"
|
||||
echo "${HOME}/.local/bin" >> "${FORGEJO_PATH}"
|
||||
"${HOME}/.local/bin/git-lfs" version
|
||||
|
||||
- name: Configure Git auth for Forgejo LFS
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "${REPO_SERVER_URL}" ]; then
|
||||
echo "forgejo.server_url is required for Forgejo LFS auth"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${REPO_NAME}" ]; then
|
||||
echo "forgejo.repository is required for Forgejo LFS auth"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${REPO_AUTH_USER}" ] || [ -z "${REPO_TOKEN}" ]; then
|
||||
echo "Forgejo LFS auth requires a username and token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git lfs install --local
|
||||
|
||||
AUTH_B64="$(printf '%s' "${REPO_AUTH_USER}:${REPO_TOKEN}" | base64 | tr -d '\n')"
|
||||
git config --local "http.${REPO_SERVER_URL}/.extraheader" "AUTHORIZATION: basic ${AUTH_B64}"
|
||||
git config --local "http.${REPO_SERVER_URL}/${REPO_NAME}.git/info/lfs/.extraheader" "AUTHORIZATION: basic ${AUTH_B64}"
|
||||
git config --local lfs.url "${REPO_SERVER_URL}/${REPO_NAME}.git/info/lfs"
|
||||
|
||||
- name: Validate Git LFS checkout for matcher binary
|
||||
- name: Resolve matcher binary from Forgejo LFS API
|
||||
run: |
|
||||
set -eu
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "git is required to validate Git LFS checkout"
|
||||
echo "git is required to resolve the matcher binary"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! git lfs version >/dev/null 2>&1; then
|
||||
echo "git-lfs is required on the runner so the processor image can include the real matcher binary"
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
apt-get install -y python3
|
||||
else
|
||||
echo "python3 is required to parse the Forgejo LFS batch response"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "${REPO_SERVER_URL}" ] || [ -z "${REPO_NAME}" ] || [ -z "${REPO_TOKEN}" ]; then
|
||||
echo "Forgejo LFS resolution requires forgejo.server_url, forgejo.repository, and an auth token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git lfs pull --include="bin/Face_Recognition_Unix/face_matcher"
|
||||
POINTER_CONTENT="$(git show HEAD:bin/Face_Recognition_Unix/face_matcher)"
|
||||
OID="$(printf '%s\n' "${POINTER_CONTENT}" | awk '/^oid sha256:/{sub(/^oid sha256:/, ""); print; exit}')"
|
||||
MATCHER_SIZE="$(printf '%s\n' "${POINTER_CONTENT}" | awk '/^size /{print $2; exit}')"
|
||||
|
||||
if [ -z "${OID}" ] || [ -z "${MATCHER_SIZE}" ]; then
|
||||
echo "Failed to read the matcher LFS pointer from HEAD"
|
||||
printf '%s\n' "${POINTER_CONTENT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
cleanup() {
|
||||
rm -rf "${TMP_DIR}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
BATCH_URL="${REPO_SERVER_URL}/${REPO_NAME}.git/info/lfs/objects/batch"
|
||||
AUTH_BASIC_PRIMARY="$(printf '%s' "${REPO_AUTH_USER}:${REPO_TOKEN}" | base64 | tr -d '\n')"
|
||||
AUTH_BASIC_ACTOR="$(printf '%s' "${REPO_ACTOR}:${REPO_TOKEN}" | base64 | tr -d '\n')"
|
||||
|
||||
cat > "${TMP_DIR}/batch-request.json" <<EOF
|
||||
{
|
||||
"operation": "download",
|
||||
"transfers": ["basic"],
|
||||
"ref": {"name": "${FORGEJO_REF}"},
|
||||
"objects": [
|
||||
{
|
||||
"oid": "${OID}",
|
||||
"size": ${MATCHER_SIZE}
|
||||
}
|
||||
],
|
||||
"hash_algo": "sha256"
|
||||
}
|
||||
EOF
|
||||
|
||||
batch_request() {
|
||||
label="$1"
|
||||
header_name="$2"
|
||||
header_value="$3"
|
||||
status="$(curl -sS -o "${TMP_DIR}/batch-response.json" -D "${TMP_DIR}/batch-headers.txt" -w '%{http_code}' \
|
||||
-X POST "${BATCH_URL}" \
|
||||
-H 'Accept: application/vnd.git-lfs+json' \
|
||||
-H 'Content-Type: application/vnd.git-lfs+json' \
|
||||
-H "${header_name}: ${header_value}" \
|
||||
--data @"${TMP_DIR}/batch-request.json")"
|
||||
echo "Batch auth attempt ${label}: HTTP ${status}"
|
||||
[ "${status}" = "200" ]
|
||||
}
|
||||
|
||||
if ! batch_request "basic-primary" "Authorization" "basic ${AUTH_BASIC_PRIMARY}"; then
|
||||
if ! batch_request "basic-actor" "Authorization" "basic ${AUTH_BASIC_ACTOR}"; then
|
||||
if ! batch_request "token" "Authorization" "token ${REPO_TOKEN}"; then
|
||||
echo "Forgejo LFS batch request failed with all auth variants"
|
||||
cat "${TMP_DIR}/batch-response.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
python3 - "${TMP_DIR}/batch-response.json" "${TMP_DIR}/download-url.txt" "${TMP_DIR}/download-headers.txt" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
response_path, url_path, headers_path = sys.argv[1:4]
|
||||
with open(response_path, 'r', encoding='utf-8') as handle:
|
||||
data = json.load(handle)
|
||||
|
||||
objects = data.get('objects') or []
|
||||
if not objects:
|
||||
raise SystemExit('Forgejo LFS batch response did not contain any objects')
|
||||
|
||||
obj = objects[0]
|
||||
if 'error' in obj:
|
||||
raise SystemExit(f"Forgejo LFS object error: {obj['error']}")
|
||||
|
||||
download = ((obj.get('actions') or {}).get('download'))
|
||||
if not download or 'href' not in download:
|
||||
raise SystemExit('Forgejo LFS batch response did not contain a download action')
|
||||
|
||||
with open(url_path, 'w', encoding='utf-8') as handle:
|
||||
handle.write(download['href'])
|
||||
|
||||
headers = download.get('header') or {}
|
||||
with open(headers_path, 'w', encoding='utf-8') as handle:
|
||||
for key, value in headers.items():
|
||||
handle.write(f'{key}: {value}\n')
|
||||
PY
|
||||
|
||||
DOWNLOAD_URL="$(cat "${TMP_DIR}/download-url.txt")"
|
||||
download_args=()
|
||||
while IFS= read -r header; do
|
||||
[ -n "${header}" ] || continue
|
||||
download_args+=(-H "${header}")
|
||||
done < "${TMP_DIR}/download-headers.txt"
|
||||
|
||||
curl -fL "${download_args[@]}" -o bin/Face_Recognition_Unix/face_matcher "${DOWNLOAD_URL}"
|
||||
|
||||
if grep -q "https://git-lfs.github.com/spec/v1" bin/Face_Recognition_Unix/face_matcher; then
|
||||
echo "bin/Face_Recognition_Unix/face_matcher is still an LFS pointer, not the real binary"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MATCHER_SIZE="$(wc -c < bin/Face_Recognition_Unix/face_matcher | tr -d '[:space:]')"
|
||||
if [ "${MATCHER_SIZE}" -lt 1000000 ]; then
|
||||
echo "bin/Face_Recognition_Unix/face_matcher is unexpectedly small (${MATCHER_SIZE} bytes) and does not look like the real binary"
|
||||
RESOLVED_SIZE="$(wc -c < bin/Face_Recognition_Unix/face_matcher | tr -d '[:space:]')"
|
||||
if [ "${RESOLVED_SIZE}" -lt 1000000 ]; then
|
||||
echo "bin/Face_Recognition_Unix/face_matcher is unexpectedly small (${RESOLVED_SIZE} bytes) and does not look like the real binary"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Validated Git LFS checkout for matcher binary (${MATCHER_SIZE} bytes)"
|
||||
if [ "${RESOLVED_SIZE}" != "${MATCHER_SIZE}" ]; then
|
||||
echo "bin/Face_Recognition_Unix/face_matcher size mismatch: expected ${MATCHER_SIZE}, got ${RESOLVED_SIZE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Resolved matcher binary from Forgejo LFS API (${RESOLVED_SIZE} bytes)"
|
||||
|
||||
- name: Validate registry secrets
|
||||
run: |
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue