From 22011ce6ac165372e34006d5c7e02fc96f5ec6d7 Mon Sep 17 00:00:00 2001 From: Netherwarlord Date: Wed, 29 Jul 2026 05:16:43 -0400 Subject: [PATCH] feat: implement Gitea CI workflows for release and testing, enhance build script for multi-platform support --- .gitea/workflows/release.yml | 59 ++++++++++++ .gitea/workflows/tests.yml | 61 ++++++++++++ README.md | 24 +++-- packaging/GITEA-SETUP.md | 152 +++++++++++++++++++++++++++++ packaging/build.py | 30 ++++-- packaging/publish_gitea.py | 180 +++++++++++++++++++++++++++++++++++ 6 files changed, 491 insertions(+), 15 deletions(-) create mode 100644 .gitea/workflows/release.yml create mode 100644 .gitea/workflows/tests.yml create mode 100644 packaging/GITEA-SETUP.md create mode 100644 packaging/publish_gitea.py diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..32722a1 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,59 @@ +name: release + +on: + push: + tags: ["v*"] + workflow_dispatch: + +jobs: + release: + # One Linux runner produces all three archives. build.py normalises + # everything that would otherwise vary by build host — file modes, tar + # ownership, and launcher line endings — so a Windows zip built here is + # equivalent to one built on Windows. That removes the three-runner matrix, + # the artifact hand-off between jobs, and the race of three jobs trying to + # create the same release. + # + # Multi-OS runners would still earn their keep for *testing*, which is a + # different question from building. + runs-on: ubuntu-latest + steps: + # build.py copies only what "git ls-files" reports, so it needs real + # history rather than a tarball export. + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + # No pip install: build.py and publish_gitea.py are both stdlib-only, so + # the release path cannot be broken by a dependency it does not ship. + - name: Build all three archives + run: python packaging/build.py --platform all + + # Tag pushes publish; a manual run stops after the build above. Guarding + # the step rather than the job is what keeps workflow_dispatch usable as + # a dry run — the archives still get built and the logs still show their + # sizes, without a release being created. + - name: Publish to Gitea + if: startsWith(github.ref, 'refs/tags/v') + env: + GITEA_URL: ${{ github.server_url }} + GITEA_REPOSITORY: ${{ github.repository }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + # Leaves a draft. Review the archives, then publish from the web UI, or + # add --publish here once you trust the pipeline. + run: >- + python packaging/publish_gitea.py + --tag "${{ github.ref_name }}" + dist/LM-Gambit-*.zip dist/LM-Gambit-*.tar.gz + + - name: Dry run note + if: ${{ !startsWith(github.ref, 'refs/tags/v') }} + run: | + echo "No tag — archives built but not published." + ls -lh dist/ diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml new file mode 100644 index 0000000..661f921 --- /dev/null +++ b/.gitea/workflows/tests.yml @@ -0,0 +1,61 @@ +name: tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + python: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # 3.11 is what this is developed against. 3.10 is the floor the README + # advertises, so it is checked rather than assumed. + python: ["3.10", "3.11"] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + # requirements-ci.txt deliberately omits llama-cpp-python: it needs a C + # compiler, dominates install time, and nothing under tests_py/ imports + # it. The tests cover suite loading, grading, reporting and the HTTP + # surface — none of which touch local inference. + - name: Install + run: python -m pip install -r requirements-ci.txt + + # test_api.py skips itself when nothing is listening on :8765, so the + # other eight files run and the suite still reports success. + - name: Run tests + run: python tests_py/run_all.py + + web: + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install + run: npm ci + + # Runs tsc as well as the bundler, so a type error fails here. + - name: Type-check and build + run: npm run build + + - name: Run tests + run: npm test diff --git a/README.md b/README.md index f000428..0654da3 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,8 @@ from one origin. ## Building a release ```bash -python packaging/build.py # archive for the current platform +python packaging/build.py --platform all # the whole release set +python packaging/build.py # just the current platform python packaging/build.py --platform linux # or windows / macos python packaging/build.py --skip-web # reuse an existing web/dist ``` @@ -483,16 +484,27 @@ Only files `git ls-files` reports get copied, plus `web/dist`. Local models, results, settings and virtual environments cannot leak into an archive even if they are sitting in the working tree. -**Via GitHub Actions** — `.github/workflows/release.yml` builds all three on -`windows-latest`, `ubuntu-latest` and `macos-latest`. Pushing a `v*` tag builds -the archives and opens a **draft** release with them attached; `workflow_dispatch` -builds and uploads artifacts without creating a release, which is the way to -test the pipeline. +### In CI + +Workflows exist for both forges. Pushing a `v*` tag produces a **draft** release +with the three archives attached; `workflow_dispatch` builds them without +creating a release, which is how to test the pipeline before a tag exists. ```bash git tag v2.0.0 && git push origin v2.0.0 ``` +| | Builds on | Publishes with | +|---|---|---| +| `.github/workflows/release.yml` | `windows-latest`, `ubuntu-latest`, `macos-latest` | `softprops/action-gh-release` | +| `.gitea/workflows/release.yml` | one `ubuntu-latest` runner | `packaging/publish_gitea.py` | + +The Gitea one needs only a single runner because the cross-build makes the +archives independent of build host — see +[packaging/GITEA-SETUP.md](packaging/GITEA-SETUP.md) for runner registration and +the `GITEA_TOKEN` secret. `publish_gitea.py` is stdlib-only and idempotent: +re-running on the same tag replaces assets rather than duplicating them. + --- ## Extending diff --git a/packaging/GITEA-SETUP.md b/packaging/GITEA-SETUP.md new file mode 100644 index 0000000..9a9beb4 --- /dev/null +++ b/packaging/GITEA-SETUP.md @@ -0,0 +1,152 @@ +# Running CI on a self-hosted Gitea + +The workflows in `.gitea/workflows/` mirror the GitHub ones in +`.github/workflows/`, with one structural difference: **releases build on a +single Linux runner** rather than three. + +That is not a compromise. `packaging/build.py` normalises everything that would +otherwise vary by build host — tar member modes, uid/gid, and launcher line +endings — so the Windows `.zip` produced on Linux is equivalent to one produced +on Windows. Dropping the matrix also drops the artifact hand-off between jobs +and the race of three jobs racing to create the same release. + +Multi-OS runners would still be worth adding for *testing*. Building does not +need them. + +--- + +## 1. Enable Actions + +In `app.ini`: + +```ini +[actions] +ENABLED = true +DEFAULT_ACTIONS_URL = https://github.com +``` + +`DEFAULT_ACTIONS_URL` is where `actions/checkout` and friends are fetched from. +Public repositories clone anonymously, so a locked or absent GitHub account does +not affect this. Restart Gitea afterwards. + +## 2. Register a runner + +Get a registration token from **Site Administration → Actions → Runners**, then: + +```bash +docker run -d --restart always --name gitea-runner \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /srv/gitea-runner:/data \ + -e GITEA_INSTANCE_URL=https://git.example.lan \ + -e GITEA_RUNNER_REGISTRATION_TOKEN= \ + -e GITEA_RUNNER_LABELS=ubuntu-latest:docker://catthehacker/ubuntu:act-22.04 \ + gitea/act_runner:latest +``` + +The label mapping is the part that matters. `runs-on: ubuntu-latest` in the +workflow resolves through it to a container image. `catthehacker/ubuntu` images +carry the tooling GitHub's hosted runners have; the bare `ubuntu:22.04` image +does not, and `actions/setup-python` will fail on it for want of basic +utilities. + +Mounting the Docker socket lets the runner start job containers. It also grants +effective root on the host, which is the usual trade — acceptable on a machine +that only serves this purpose, worth knowing about regardless. + +## 3. Add the release token + +Releases are published through Gitea's own API by +`packaging/publish_gitea.py`, so the job needs a token: + +1. **Settings → Applications → Generate New Token**, scope `write:repository`. +2. In the repository: **Settings → Actions → Secrets → Add Secret**, named + `GITEA_TOKEN`. + +`GITEA_URL` and `GITEA_REPOSITORY` come from `github.server_url` and +`github.repository`, which Gitea populates for compatibility. Nothing else to +configure. + +## 4. If the runner cannot reach Gitea + +Worth understanding before it bites, because the failure is confusing. + +`ROOT_URL` is what Gitea advertises to the world, and job containers inherit it +as `github.server_url` — so `actions/checkout` and `publish_gitea.py` both call +the *public* address, even though the runner may be sitting on the same host as +Gitea. When that address only resolves outside your network, jobs fail at +checkout with a DNS or connection error while Gitea itself is plainly fine. + +Two ways out. + +**Let it hairpin.** If your router does NAT loopback, the request leaves and +comes straight back to the reverse proxy, and nothing needs configuring. Try +this first — it is the common case and costs nothing to test. + +**Pin the name to a local address.** Map the public hostname to your reverse +proxy's LAN address, on the runner and on the containers it spawns: + +```yaml +services: + gitea-runner: + extra_hosts: + - "git.example.lan:192.0.2.10" # the reverse proxy, not Gitea +``` + +and in act_runner's `config.yaml`, so job containers inherit it: + +```yaml +container: + options: --add-host=git.example.lan:192.0.2.10 +``` + +The trap is which host to point at. It must be whatever terminates the scheme +in `ROOT_URL`. If `ROOT_URL` is `https://…` then the override has to reach +something serving TLS on 443 — your reverse proxy. Pointing it straight at the +Gitea host sends an HTTPS request to a port where Gitea is only speaking plain +HTTP, and the connection is refused. + +Also: do not put Gitea behind a forward-auth proxy (Authentik outposts, +oauth2-proxy and friends). Runners authenticate with their own token and cannot +satisfy an SSO interstitial, so registration and every job will fail. Use the +identity provider as an OIDC source *inside* Gitea's own login instead. + +## 5. Cut a release + +Dry run first — `workflow_dispatch` builds all three archives and prints their +sizes without creating anything: + +**Actions → release → Run workflow** + +Then tag: + +```bash +git tag v2.0.0 && git push origin v2.0.0 +``` + +That leaves a **draft** release with the three archives attached. Review, then +publish from the web UI. Re-running the workflow on the same tag replaces the +assets rather than duplicating them, so a half-finished publish is safe to +retry. + +To publish immediately instead of drafting, add `--publish` to the +`publish_gitea.py` invocation in `.gitea/workflows/release.yml`. + +--- + +## Notes for a DL380p Gen8 + +Those E5-2600 v1/v2 chips are **x86-64-v2** — they have AVX but not AVX2, which +arrived with the v3 (Haswell) generation. Two consequences: + +- **Pick the distribution with that in mind.** Debian, Ubuntu and RHEL 9 target + a baseline these CPUs meet. RHEL 10 and recent Fedora raised their floor to + x86-64-v3 and will not run. Node 22 and Docker are fine either way. + +- **Don't plan to run benchmarks on this box.** Many prebuilt + `llama-cpp-python` wheels assume AVX2, and the ones that don't will be slow + on 2012 silicon regardless. It is a fine CI and git server; keep inference on + the workstation. Nothing in `tests_py/` needs `llama-cpp-python`, which is + exactly why `requirements-ci.txt` omits it. + +Two dual-socket Sandy/Ivy Bridge sockets have plenty of throughput for Gitea +plus a runner — this is an I/O and RAM workload, not a vector-math one. diff --git a/packaging/build.py b/packaging/build.py index cbe46c5..48fa189 100644 --- a/packaging/build.py +++ b/packaging/build.py @@ -192,23 +192,35 @@ def _normalize(info: tarfile.TarInfo) -> tarfile.TarInfo: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--platform", choices=sorted(PLATFORMS), default=current_platform()) + parser.add_argument( + "--platform", + choices=[*sorted(PLATFORMS), "all"], + default=current_platform(), + help="Target platform, or 'all' for a complete release set", + ) parser.add_argument("--skip-web", action="store_true", help="Reuse an existing web/dist") args = parser.parse_args() - version = read_version() - print(f"LM-Gambit {version} -> {args.platform}") + targets = sorted(PLATFORMS) if args.platform == "all" else [args.platform] + version = read_version() + print(f"LM-Gambit {version} -> {', '.join(targets)}") + + # Once, not once per target: the bundle is identical on every platform. if not args.skip_web: build_web() - staging = DIST / f"_staging-{args.platform}" - stage(args.platform, staging) - out = archive(args.platform, staging, version) - shutil.rmtree(staging, ignore_errors=True) + built = [] + for target in targets: + staging = DIST / f"_staging-{target}" + stage(target, staging) + built.append(archive(target, staging, version)) + shutil.rmtree(staging, ignore_errors=True) - size_mb = out.stat().st_size / (1024 * 1024) - print(f"\n {out.relative_to(ROOT)} ({size_mb:.1f} MB)") + print() + for out in built: + size_mb = out.stat().st_size / (1024 * 1024) + print(f" {out.relative_to(ROOT)} ({size_mb:.1f} MB)") return 0 diff --git a/packaging/publish_gitea.py b/packaging/publish_gitea.py new file mode 100644 index 0000000..35750da --- /dev/null +++ b/packaging/publish_gitea.py @@ -0,0 +1,180 @@ +"""Publish release archives to a Gitea instance. + + python packaging/publish_gitea.py --tag v2.0.0 dist/*.zip dist/*.tar.gz + +Creates the release if it does not exist, then uploads each file as an asset. +Re-running replaces assets of the same name rather than erroring, so a failed +half-finished publish can simply be run again. + +Reads configuration from the environment, which Gitea Actions populates: + + GITEA_URL instance root, e.g. https://git.example.lan + (falls back to GITHUB_SERVER_URL, which Gitea sets) + GITEA_REPOSITORY "owner/repo" (falls back to GITHUB_REPOSITORY) + GITEA_TOKEN an access token with write:repository + +**Why not a marketplace action.** ``softprops/action-gh-release`` targets +GitHub's API and does not work against Gitea. The Gitea-flavoured forks exist +but pin you to a third party for the one step that matters most, on a forge +whose API is small enough to call directly. urllib covers it in ~150 lines with +nothing to install and nothing to trust. +""" + +from __future__ import annotations + +import argparse +import json +import mimetypes +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +TIMEOUT = 120 + + +class PublishError(RuntimeError): + pass + + +def _request( + method: str, + url: str, + token: str, + *, + body: bytes | None = None, + content_type: str | None = None, +) -> dict | list | None: + request = urllib.request.Request(url, data=body, method=method) + request.add_header("Authorization", f"token {token}") + request.add_header("Accept", "application/json") + if content_type: + request.add_header("Content-Type", content_type) + + try: + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: + payload = response.read() + except urllib.error.HTTPError as error: + # The response body carries Gitea's actual complaint; without it the + # caller only sees "HTTP Error 422" and has to guess. + detail = error.read().decode("utf-8", "replace").strip() + raise PublishError(f"{method} {url} -> {error.code} {error.reason}\n{detail}") from None + except urllib.error.URLError as error: + raise PublishError(f"{method} {url} -> {error.reason}") from None + + return json.loads(payload) if payload else None + + +def find_or_create_release(api: str, token: str, tag: str, *, draft: bool, notes: str) -> dict: + try: + return _request("GET", f"{api}/releases/tags/{tag}", token) + except PublishError as error: + if "404" not in str(error): + raise + + print(f"Creating release {tag}") + body = json.dumps( + {"tag_name": tag, "name": tag, "body": notes, "draft": draft, "prerelease": False} + ).encode() + return _request("POST", f"{api}/releases", token, body=body, content_type="application/json") + + +def upload_asset(api: str, token: str, release: dict, path: Path) -> None: + # Uploading over an existing name appends rather than replaces, which would + # leave two assets called LM-Gambit-2.0.0-linux.tar.gz after a re-run. + for asset in release.get("assets") or []: + if asset.get("name") == path.name: + print(f" replacing existing {path.name}") + _request("DELETE", f"{api}/releases/{release['id']}/assets/{asset['id']}", token) + + boundary = f"----LMGambit{uuid.uuid4().hex}" + mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + body = b"".join( + [ + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="attachment"; filename="{path.name}"\r\n'.encode(), + f"Content-Type: {mime}\r\n\r\n".encode(), + path.read_bytes(), + f"\r\n--{boundary}--\r\n".encode(), + ] + ) + + url = f"{api}/releases/{release['id']}/assets?name={urllib.parse.quote(path.name)}" + _request( + "POST", + url, + token, + body=body, + content_type=f"multipart/form-data; boundary={boundary}", + ) + print(f" uploaded {path.name} ({path.stat().st_size / (1024 * 1024):.1f} MB)") + + +DEFAULT_NOTES = """\ +Download the archive for your platform, extract it, and run the launcher +inside — `LM-Gambit.bat` on Windows, `./LM-Gambit.sh` on macOS and Linux. + +The first run creates a virtual environment and installs dependencies; that +takes a few minutes and happens once. It needs **Python 3.10 or newer** on the +machine. See `GPU-ACCELERATION.md` in the archive for CUDA, Metal and ROCm. + +Scores read as *"correct where checkable, well-formed elsewhere"* — 16 of the +26 built-in questions have answer keys, and the rest are graded on form. +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("files", nargs="+", type=Path, help="Archives to attach") + parser.add_argument("--tag", required=True, help="Tag to release, e.g. v2.0.0") + parser.add_argument( + "--publish", + action="store_true", + help="Publish immediately instead of leaving a draft for review", + ) + args = parser.parse_args() + + url = os.environ.get("GITEA_URL") or os.environ.get("GITHUB_SERVER_URL") + repository = os.environ.get("GITEA_REPOSITORY") or os.environ.get("GITHUB_REPOSITORY") + token = os.environ.get("GITEA_TOKEN") + + missing = [ + name + for name, value in ( + ("GITEA_URL", url), + ("GITEA_REPOSITORY", repository), + ("GITEA_TOKEN", token), + ) + if not value + ] + if missing: + print(f"Missing environment: {', '.join(missing)}", file=sys.stderr) + return 1 + + missing_files = [str(path) for path in args.files if not path.is_file()] + if missing_files: + print(f"No such file: {', '.join(missing_files)}", file=sys.stderr) + return 1 + + api = f"{url.rstrip('/')}/api/v1/repos/{repository}" + + try: + release = find_or_create_release( + api, token, args.tag, draft=not args.publish, notes=DEFAULT_NOTES + ) + for path in args.files: + upload_asset(api, token, release, path) + except PublishError as error: + print(f"\n{error}", file=sys.stderr) + return 1 + + state = "published" if args.publish else "draft" + print(f"\n{args.tag} {state}: {url.rstrip('/')}/{repository}/releases/tag/{args.tag}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())