From 18054331d97e913a601ae07235c3bd2a6bcd7099 Mon Sep 17 00:00:00 2001 From: Zsolt Tasnadi Date: Sat, 15 Aug 2026 21:29:05 +0200 Subject: [PATCH] general store infra --- README.md | 177 ++++++++++++++++------- config.example.json | 39 +++++ install.sh | 112 ++++++++++----- ttg-store => store.py | 328 ++++++++++++++++++++++++++++-------------- 4 files changed, 464 insertions(+), 192 deletions(-) create mode 100644 config.example.json rename ttg-store => store.py (70%) diff --git a/README.md b/README.md index 67c1a99..2feaafa 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,158 @@ -# ttg-store — Teletype Games catalog client for Batocera +# warp-engine-batocera-store — a Batocera store engine for WarpEngine sites -Pulls games from the WarpEngine catalog on straight -into a Batocera box's ROM folders, with EmulationStation metadata and box art. -Triggered from the **Ports** menu on the device, or from the CLI over SSH. +Pulls games from a [WarpEngine](https://git.teletypegames.org/tools/warp_engine) +catalog straight into a Batocera box's ROM folders, with EmulationStation +metadata and box art. Triggered from the **Ports** menu on the device, or from +the CLI over SSH. + +This repository is the engine only. It knows the WarpEngine API but nothing +about any particular site: the host, the store's name and where its games land +all come from a `config.json` that lives in a separate **store repository**. ``` -Ports ▸ "Teletype Games Store" +warp-engine-batocera-store the engine — store.py, install.sh + ▲ + │ config.json + │ + ┌─────┴─────┬───────────────┐ + │ │ │ +ttg-… my-… other-… store repositories + │ │ │ + ▼ ▼ ▼ +teletypegames.org my.example other.example +``` + +The reference store is +[`ttg-batocera-store`](https://git.teletypegames.org/tools/ttg-batocera-store). + +``` +Ports ▸ "" │ ├─ GET /api/software the whole catalog ├─ keep platforms this box can run c64 → c64, tic80 → tic80 ├─ pick the newest non-dev release that carries the right asset ├─ GET /api/download?path= .prg / .tic into the ROM folder - ├─ GET /api/image/ box art + ├─ GET box art └─ merge gamelist.xml title, desc, author, image ``` Python 3 standard library only — Batocera ships python3 and no pip. -Repository: `https://git.teletypegames.org/tools/batocera-store`. -The catalog side lives in `teletypegames` (`libs/ruby/warp_engine`). +## Installing a store -## Install - -From a checkout of this repo: +The installer takes the store as a parameter, so it works with any config: ```sh -scp -r batocera-store root@batocera:/tmp/ -ssh root@batocera /tmp/batocera-store/install.sh +STORE_CONFIG=https://git.example.org/tools/my-store/raw/branch/master/config.json \ + bash <(curl -fsSL https://git.teletypegames.org/tools/warp-engine-batocera-store/raw/branch/master/install.sh) ``` -Or straight onto the device: +From a checkout, with a local config: ```sh -curl -fsSL https://git.teletypegames.org/tools/batocera-store/raw/branch/master/install.sh | bash +scp -r warp-engine-batocera-store root@batocera:/tmp/ +ssh root@batocera 'STORE_CONFIG=/tmp/my-config.json /tmp/warp-engine-batocera-store/install.sh' ``` -The installer drops `ttg-store` in `/userdata/system/ttg-store/`, writes a -default config, creates the Ports entry and runs the first sync. Restart -EmulationStation (`batocera-es-swissknife --restart`) to see the games. +It reads `store.id` from the config, installs the engine and the config into +`/userdata/system/batocera-store//`, writes a `-store` launcher and a +Ports entry, and runs the first sync. Restart EmulationStation +(`batocera-es-swissknife --restart`) to see the games. + +Installer knobs, all environment variables: `STORE_CONFIG` (required), +`BATOCERA_STORE_ROOT`, `BATOCERA_PORTS_DIR`, `BATOCERA_PORT_NAME`, +`ENGINE_RAW_BASE`. + +## Writing a store repository + +Three files: + +``` +my-batocera-store/ +├── config.json the store: URL, name, subfolder, platform mapping +├── install.sh a wrapper that hands that config to the engine installer +└── README.md +``` + +`install.sh` is small enough to quote in full: + +```sh +#!/bin/bash +set -euo pipefail +ENGINE_RAW_BASE="${ENGINE_RAW_BASE:-https://git.teletypegames.org/tools/warp-engine-batocera-store/raw/branch/master}" +export STORE_CONFIG="${STORE_CONFIG:-https://git.example.org/tools/my-batocera-store/raw/branch/master/config.json}" +export BATOCERA_PORT_NAME="My Store" +curl -fsSL "$ENGINE_RAW_BASE/install.sh" | bash +``` + +Pick a `store.id` and a `paths.subfolder` nobody else uses — they are what keeps +two stores on the same box from deleting each other's games. ## Use -**On the device.** *Ports ▸ Teletype Games Store*. The script downloads -anything new, then restarts EmulationStation so the games show up. Ports -scripts get no console in Batocera, so the output goes to -`/userdata/system/ttg-store/ttg-store.log`. +**On the device.** *Ports ▸ "\"*. The script downloads anything +new, then restarts EmulationStation so the games show up. Ports scripts get no +console in Batocera, so the output goes to `store.log` in the store home. -**Over SSH.** +**Over SSH**, through the launcher the installer wrote: ```sh -ttg-store list # compatible catalog entries; * installed, ^ update available -ttg-store sync # download everything new, refresh gamelists -ttg-store sync blessingofra # just one title (never prunes) -ttg-store -n sync # dry run -ttg-store remove c64demo # uninstall -ttg-store config # effective configuration +S=/userdata/system/batocera-store/example-store + +$S list # compatible catalog entries; * installed, ^ update available +$S sync # download everything new, refresh gamelists +$S sync blessingofra # just one title (never prunes) +$S -n sync # dry run +$S remove c64:demo # uninstall (bare name works too) +$S config # effective configuration ``` -Global flags go **before** the subcommand: `ttg-store --roms-root /tmp/roms sync`. +Global flags go **before** the subcommand: `$S --roms-root /tmp/roms sync`. ## Where things land ``` -/userdata/system/ttg-store/ ttg-store, config.json, state.json, catalog.json, ttg-store.log -/userdata/roms/c64/teletypegames/ blessingofra-2.0.0.prg, rabbit-1.0.0.prg, … -/userdata/roms/c64/teletypegames/images/ blessingofra.png, … -/userdata/roms/c64/gamelist.xml our entries merged in -/userdata/roms/ports/Teletype Games Store.sh +/userdata/system/batocera-store/ +├── example-store launcher for the store below +└── example/ + ├── store.py, config.json the engine and the store that selected it + └── state.json, catalog.json, store.log + +/userdata/roms/c64/example/ blessingofra-2.0.0.prg, rabbit-1.0.0.prg, … +/userdata/roms/c64/example/images/ blessingofra.png, … +/userdata/roms/c64/gamelist.xml our entries merged in +/userdata/roms/ports/Example Store.sh ``` -Everything installed lives under the `teletypegames/` subfolder of a system, so -a prune can never reach ROMs you put there yourself. The first time a -`gamelist.xml` is touched it is copied to `gamelist.xml.ttg-backup`; on every -merge only ``/`` nodes under `teletypegames/` are rewritten — -your own entries keep their play counts, favourites and scraped media. +Everything a store installs lives under its own `paths.subfolder`, so a prune +can never reach ROMs you put there yourself — or another store's games. The +first time a `gamelist.xml` is touched it is copied to +`gamelist.xml.-backup`; on every merge only ``/`` nodes +under that store's subfolder are rewritten, so your own entries keep their play +counts, favourites and scraped media, and two stores can share one gamelist. ## Configuration -`/userdata/system/ttg-store/config.json`, written by `ttg-store config --write`: +`config.json` in the store home, template in `config.example.json`: | Key | Default | Meaning | | --- | --- | --- | -| `base_url` | `https://teletypegames.org` | WarpEngine host | -| `roms_root` | `/userdata/roms` | where systems live | -| `subfolder` | `teletypegames` | our subfolder inside each system | -| `folder_name` | `Teletype Games` | display name of that folder in ES | -| `statuses` | `["released", "archived"]` | catalog `status` values to install | -| `owner_id` | `null` | restrict to one publisher (`/api/software?owner_id=`) | -| `only` / `exclude` | `[]` | software-name allow / deny lists | +| `store.id` | `warp` | slug: names the store home, log prefix and gamelist backup | +| `store.name` | `WarpEngine Store` | display name for the Ports entry | +| `store.base_url` | — | WarpEngine host | +| `store.api.catalog` | `/api/software` | catalog endpoint, if the engine is mounted elsewhere | +| `store.api.download` | `/api/download` | download endpoint | +| `paths.roms_root` | `/userdata/roms` | where systems live | +| `paths.subfolder` | `warp` | this store's subfolder inside each system | +| `emulationstation.folder_name` | `WarpEngine Store` | display name of that folder in ES | +| `emulationstation.restart` | `true` | restart EmulationStation after a change | +| `catalog.statuses` | `["released", "archived"]` | catalog `status` values to install | +| `catalog.owner_id` | `null` | restrict to one publisher (`/api/software?owner_id=`) | +| `catalog.only` / `catalog.exclude` | `[]` | software-name allow / deny lists | | `platforms` | c64, tic80 | platform → system, asset kind, extension, enabled | -| `prune` | `true` | remove games that left the catalog or the filters | -| `restart_es` | `true` | restart EmulationStation after a change | -| `timeout` | `30` | HTTP timeout, seconds | -| `insecure` | `false` | skip TLS verification (self-hosted test instances) | +| `behavior.prune` | `true` | remove games that left the catalog or the filters | +| `behavior.timeout` | `30` | HTTP timeout, seconds | +| `behavior.insecure` | `false` | skip TLS verification (self-hosted test instances) | ### Platform mapping @@ -106,20 +166,20 @@ system can boot directly: The other WarpEngine platforms (`ebitengine`, `love`, `godot`, `bevy`, `phaser`) ship `html` and per-OS zip archives, not a ROM a Batocera system -launches, so they are not mapped. Adding one is a config edit: +launches, so they are not mapped by default. Adding one is a config edit: ```json "platforms": { "godot": { "system": "godot", "kind": "linux_x64", "ext": ".zip", "enabled": true } } ``` A platform is also skipped when the box has no ROM folder for its system — -`ttg-store list` reports that as `skipped : no '' ROM folder`. +`list` reports that as `skipped : no '' ROM folder`. ## How a sync stays safe next to EmulationStation EmulationStation keeps gamelists in memory and writes them back when it exits, so a merge done while it runs can be clobbered. Launched from the Ports menu -(`sync-from-es`), the tool therefore downloads first, then hands the gamelist +(`sync-from-es`), the engine therefore downloads first, then hands the gamelist merge to a detached `apply-gamelists --wait-pid ` child that waits for the old ES process to die before writing. From SSH, with no ES running, the merge happens inline. @@ -128,3 +188,10 @@ Re-running a sync is idempotent: files already present are left alone, and a release bump (`1.1` → `2.0.0`) deletes the old asset before fetching the new one. `GET /api/download` is used rather than `/file/`, so downloads count towards the catalog's statistics. + +## State + +`state.json` records what this store installed, keyed `:` so the +same software name on two systems stays two entries. A `version: 1` state file +written by the pre-split `ttg-store` is migrated to that key format on first +run. diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..a61726f --- /dev/null +++ b/config.example.json @@ -0,0 +1,39 @@ +{ + "store": { + "id": "example", + "name": "Example Store", + "base_url": "https://example.org", + "api": { + "catalog": "/api/software", + "download": "/api/download" + } + }, + + "paths": { + "roms_root": "/userdata/roms", + "subfolder": "example" + }, + + "emulationstation": { + "folder_name": "Example Store", + "restart": true + }, + + "catalog": { + "statuses": ["released", "archived"], + "owner_id": null, + "only": [], + "exclude": [] + }, + + "platforms": { + "c64": { "system": "c64", "kind": "cartridge", "ext": ".prg", "enabled": true }, + "tic80": { "system": "tic80", "kind": "cartridge", "ext": ".tic", "enabled": true } + }, + + "behavior": { + "prune": true, + "timeout": 30, + "insecure": false + } +} diff --git a/install.sh b/install.sh index 8b8660c..2297cc5 100755 --- a/install.sh +++ b/install.sh @@ -1,70 +1,116 @@ #!/bin/bash -# Install ttg-store on a Batocera box. +# Install a WarpEngine store on a Batocera box. # -# scp -r batocera-store root@batocera:/tmp/ && ssh root@batocera /tmp/batocera-store/install.sh +# This installer belongs to the engine, so it takes the store as a parameter: +# point STORE_CONFIG at a config.json — a local path or a URL — and it sets up +# that store. A store repository's own install.sh is a three-line wrapper that +# supplies its config; see README. # -# or, straight from the forge: -# -# curl -fsSL https://git.teletypegames.org/tools/batocera-store/raw/branch/master/install.sh | bash +# STORE_CONFIG=./config.json ./install.sh +# STORE_CONFIG=https://git.example.org/tools/my-store/raw/branch/master/config.json \ +# bash <(curl -fsSL https://git.teletypegames.org/tools/warp-engine-batocera-store/raw/branch/master/install.sh) # set -euo pipefail -HOME_DIR="${TTG_STORE_HOME:-/userdata/system/ttg-store}" -PORTS_DIR="${TTG_PORTS_DIR:-/userdata/roms/ports}" -PORT_NAME="${TTG_PORT_NAME:-Teletype Games Store}" -RAW_BASE="${TTG_RAW_BASE:-https://git.teletypegames.org/tools/batocera-store/raw/branch/master}" -SRC_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)" +STORE_ROOT="${BATOCERA_STORE_ROOT:-/userdata/system/batocera-store}" +PORTS_DIR="${BATOCERA_PORTS_DIR:-/userdata/roms/ports}" +ENGINE_RAW_BASE="${ENGINE_RAW_BASE:-https://git.teletypegames.org/tools/warp-engine-batocera-store/raw/branch/master}" +STORE_CONFIG="${STORE_CONFIG:-}" +# Only set when run as a file; piped from curl "$0" is `bash`, and then the +# engine has to come from the forge rather than from whatever the cwd holds. +SRC_DIR="" +if [ -f "$0" ]; then + SRC_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)" +fi say() { echo "[install] $*"; } +die() { echo "[install] error: $*" >&2; exit 1; } -command -v python3 >/dev/null 2>&1 || { echo "python3 is required" >&2; exit 1; } +command -v python3 >/dev/null 2>&1 || die "python3 is required" +[ -n "$STORE_CONFIG" ] || die "STORE_CONFIG is required (path or URL of a store config.json)" [ -d /userdata ] || say "warning: /userdata not found — this does not look like a Batocera box" -mkdir -p "$HOME_DIR" "$PORTS_DIR" +fetch() { # fetch ; source may be a local path or a URL + case "$1" in + http://*|https://*) curl -fsSL "$1" -o "$2" ;; + *) [ -f "$1" ] || die "no such file: $1"; cp "$1" "$2" ;; + esac +} -if [ -f "$SRC_DIR/ttg-store" ]; then - say "installing ttg-store from $SRC_DIR" - install -m 0755 "$SRC_DIR/ttg-store" "$HOME_DIR/ttg-store" +# Stage the config first: the store id in it decides where everything lands. +TMP_CFG="$(mktemp)" +trap 'rm -f "$TMP_CFG"' EXIT +say "reading store config from $STORE_CONFIG" +fetch "$STORE_CONFIG" "$TMP_CFG" + +STORE_META="$(python3 -c ' +import json, sys +cfg = json.load(open(sys.argv[1])).get("store") or {} +sid = cfg.get("id") or "" +if not sid or "/" in sid: + sys.exit("store.id is required and must not contain a slash") +print(sid, cfg.get("name") or sid) +' "$TMP_CFG")" || die "cannot read $STORE_CONFIG" + +STORE_ID="${STORE_META%% *}" +STORE_NAME="${STORE_META#* }" + +STORE_HOME="$STORE_ROOT/$STORE_ID" +PORT_NAME="${BATOCERA_PORT_NAME:-$STORE_NAME}" +LAUNCHER="$STORE_ROOT/$STORE_ID-store" + +mkdir -p "$STORE_HOME" "$PORTS_DIR" + +if [ -n "$SRC_DIR" ] && [ -f "$SRC_DIR/store.py" ]; then + say "installing the engine from $SRC_DIR" + install -m 0755 "$SRC_DIR/store.py" "$STORE_HOME/store.py" else - say "downloading ttg-store from $RAW_BASE" - curl -fsSL "$RAW_BASE/ttg-store" -o "$HOME_DIR/ttg-store" - chmod 0755 "$HOME_DIR/ttg-store" + say "downloading the engine from $ENGINE_RAW_BASE" + curl -fsSL "$ENGINE_RAW_BASE/store.py" -o "$STORE_HOME/store.py" + chmod 0755 "$STORE_HOME/store.py" fi -# ttg-store reads TTG_STORE_HOME for its config/state/cache locations. -export TTG_STORE_HOME="$HOME_DIR" - -if [ -f "$HOME_DIR/config.json" ]; then - say "keeping existing $HOME_DIR/config.json" -else - "$HOME_DIR/ttg-store" config --write +# The config is the store's identity, so it is always refreshed; state.json, +# catalog.json and the log stay put, which makes re-running the upgrade path. +if [ -f "$STORE_HOME/config.json" ]; then + say "updating $STORE_HOME/config.json (previous copy kept as config.json.bak)" + cp "$STORE_HOME/config.json" "$STORE_HOME/config.json.bak" fi +install -m 0644 "$TMP_CFG" "$STORE_HOME/config.json" + +# A short launcher so the CLI is one path, not a python invocation. +cat > "$LAUNCHER" < "$PORTS_DIR/$PORT_NAME.sh" <>"$HOME_DIR/ttg-store.log" 2>&1 +# $STORE_NAME — syncs the catalog into this box's ROM folders. +exec >>"$STORE_HOME/store.log" 2>&1 echo "=== \$(date -Iseconds) sync started ===" -"$HOME_DIR/ttg-store" sync-from-es +"$LAUNCHER" sync-from-es echo "=== \$(date -Iseconds) sync finished (exit \$?) ===" EOF chmod 0755 "$PORTS_DIR/$PORT_NAME.sh" say "ports entry: $PORTS_DIR/$PORT_NAME.sh" say "running the first sync" -"$HOME_DIR/ttg-store" --verbose sync --no-restart +"$LAUNCHER" --verbose sync --no-restart cat < "$PORT_NAME" (re-run any time to pull new releases) - CLI: $HOME_DIR/ttg-store list|sync|remove - Config: $HOME_DIR/config.json - Log: $HOME_DIR/ttg-store.log + CLI: $LAUNCHER list|sync|remove + Config: $STORE_HOME/config.json + Log: $STORE_HOME/store.log Restart EmulationStation to see the games: batocera-es-swissknife --restart EOF diff --git a/ttg-store b/store.py similarity index 70% rename from ttg-store rename to store.py index 332659b..eb70e28 100755 --- a/ttg-store +++ b/store.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 -"""ttg-store — Teletype Games catalog client for Batocera. +"""store.py — WarpEngine catalog client for Batocera. -Reads the WarpEngine catalog API (`GET /api/software`), keeps the entries whose +Reads a WarpEngine catalog API (`GET /api/software`), keeps the entries whose platform maps to a system this Batocera box actually has, downloads the matching release asset into that system's ROM folder and writes EmulationStation metadata (title, description, author, box art) into its gamelist.xml. +The engine knows nothing about any particular site: which host to talk to, what +the store is called and where its games land all come from `config.json`. A +store repository is therefore just that config plus an installer — see README. + Standard library only — Batocera ships python3 but no pip packages. """ @@ -22,41 +26,62 @@ import urllib.parse import urllib.request import xml.etree.ElementTree as ET -VERSION = "1.0.0" -USER_AGENT = f"ttg-store/{VERSION} (batocera)" -STATE_VERSION = 1 +VERSION = "2.0.0" +STATE_VERSION = 2 -HOME = os.environ.get("TTG_STORE_HOME", "/userdata/system/ttg-store") -CONFIG_PATH = os.path.join(HOME, "config.json") -STATE_PATH = os.path.join(HOME, "state.json") -CATALOG_CACHE = os.path.join(HOME, "catalog.json") +# Every store keeps its code, config, state and log in one directory, so a box +# can carry several stores side by side without them treading on each other. +# The installer exports BATOCERA_STORE_HOME; run straight from a checkout the +# script's own directory is the store home. +DEFAULT_HOME = os.environ.get("BATOCERA_STORE_HOME") or os.path.dirname(os.path.abspath(__file__)) DEFAULT_CONFIG = { - # WarpEngine host serving /api/software, /api/download and /api/image. - "base_url": "https://teletypegames.org", - "roms_root": "/userdata/roms", - # Everything we install lives under this subfolder of the system's ROM dir, - # so a prune can never touch ROMs the user put there themselves. - "subfolder": "teletypegames", - "folder_name": "Teletype Games", - # Catalog `status` values worth installing. "development" is left out. - "statuses": ["released", "archived"], - # Restrict to one publisher (WarpEngine owner_id), null = whole catalog. - "owner_id": None, - # Software `name` allow/deny lists; empty allow list means "everything". - "only": [], - "exclude": [], + "store": { + # Short slug: names the state, the log prefix and the gamelist backup. + "id": "warp", + # Human-readable, used for the Ports entry and the ES folder label. + "name": "WarpEngine Store", + # The WarpEngine host serving the catalog. + "base_url": "https://example.org", + # Endpoint paths, in case the engine is mounted somewhere other than /. + "api": { + "catalog": "/api/software", + "download": "/api/download", + }, + }, + "paths": { + "roms_root": "/userdata/roms", + # Everything we install lives under this subfolder of the system's ROM + # dir, so a prune can never touch ROMs the user put there themselves — + # nor games installed by another store. + "subfolder": "warp", + }, + "emulationstation": { + # Display name of our subfolder in ES; null hides the folder node. + "folder_name": "WarpEngine Store", + # Restart EmulationStation after a sync that changed something. + "restart": True, + }, + "catalog": { + # Catalog `status` values worth installing. "development" is left out. + "statuses": ["released", "archived"], + # Restrict to one publisher (WarpEngine owner_id), null = whole catalog. + "owner_id": None, + # Software `name` allow/deny lists; empty allow list means "everything". + "only": [], + "exclude": [], + }, # WarpEngine platform -> Batocera system + which release asset to pull. "platforms": { "c64": {"system": "c64", "kind": "cartridge", "ext": ".prg", "enabled": True}, "tic80": {"system": "tic80", "kind": "cartridge", "ext": ".tic", "enabled": True}, }, - # Drop installed games that fell out of the catalog or the filters. - "prune": True, - # Restart EmulationStation after a sync that changed something. - "restart_es": True, - "timeout": 30, - "insecure": False, + "behavior": { + # Drop installed games that fell out of the catalog or the filters. + "prune": True, + "timeout": 30, + "insecure": False, + }, } @@ -65,10 +90,26 @@ DEFAULT_CONFIG = { # -------------------------------------------------------------------------- VERBOSE = False +# Log prefix and temp-file prefix; replaced with the store id once configured. +TAG = "batocera-store" + +HOME = DEFAULT_HOME +CONFIG_PATH = os.path.join(HOME, "config.json") +STATE_PATH = os.path.join(HOME, "state.json") +CATALOG_CACHE = os.path.join(HOME, "catalog.json") + + +def set_home(config_path): + """Anchor state and cache next to the config file that selected this store.""" + global HOME, CONFIG_PATH, STATE_PATH, CATALOG_CACHE + CONFIG_PATH = os.path.abspath(config_path) + HOME = os.path.dirname(CONFIG_PATH) + STATE_PATH = os.path.join(HOME, "state.json") + CATALOG_CACHE = os.path.join(HOME, "catalog.json") def log(msg): - print(f"[ttg-store] {msg}", flush=True) + print(f"[{TAG}] {msg}", flush=True) def debug(msg): @@ -77,7 +118,7 @@ def debug(msg): def die(msg, code=1): - print(f"[ttg-store] error: {msg}", file=sys.stderr, flush=True) + print(f"[{TAG}] error: {msg}", file=sys.stderr, flush=True) sys.exit(code) @@ -100,7 +141,7 @@ def write_atomic(path, blob): """Write bytes to `path` via a temp file in the same dir, then rename.""" directory = os.path.dirname(path) or "." os.makedirs(directory, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=directory, prefix=".ttg-", suffix=".tmp") + fd, tmp = tempfile.mkstemp(dir=directory, prefix=f".{TAG}-", suffix=".tmp") try: with os.fdopen(fd, "wb") as fh: fh.write(blob) @@ -109,33 +150,75 @@ def write_atomic(path, blob): if os.path.exists(tmp): os.unlink(tmp) raise + return path + + +def deep_merge(base, overlay): + """Recursively merge `overlay` into a copy of `base`; dicts merge, rest wins.""" + merged = dict(base) + for key, value in (overlay or {}).items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = deep_merge(merged[key], value) + else: + merged[key] = value + return merged def load_config(path, overrides=None): - cfg = json.loads(json.dumps(DEFAULT_CONFIG)) # deep copy + global TAG + user = load_json(path, default=None) - if user: - platforms = user.pop("platforms", None) - cfg.update(user) - if platforms: - for name, spec in platforms.items(): - merged = dict(cfg["platforms"].get(name, {})) - merged.update(spec) - cfg["platforms"][name] = merged + if user is None: + log(f"warning: no config at {path} — falling back to the built-in defaults") + cfg = deep_merge(json.loads(json.dumps(DEFAULT_CONFIG)), user or {}) + for key, value in (overrides or {}).items(): - if value is not None: - cfg[key] = value - cfg["base_url"] = cfg["base_url"].rstrip("/") + if value is None: + continue + section, _, leaf = key.partition(".") + cfg[section][leaf] = value + + cfg["store"]["base_url"] = str(cfg["store"]["base_url"]).rstrip("/") + if not cfg["store"].get("id"): + die("config: store.id is required") + if not cfg["paths"].get("subfolder"): + die("config: paths.subfolder is required — it is what keeps stores apart") + TAG = f"{cfg['store']['id']}-store" return cfg +# Shorthands for the values the code below reaches for constantly. +def base_url(cfg): + return cfg["store"]["base_url"] + + +def subfolder(cfg): + return cfg["paths"]["subfolder"] + + +def roms_root(cfg): + return cfg["paths"]["roms_root"] + + +def api_url(cfg, endpoint, params=None): + path = cfg["store"]["api"][endpoint] + url = base_url(cfg) + "/" + path.lstrip("/") + if params: + url += "?" + urllib.parse.urlencode(params) + return url + + # -------------------------------------------------------------------------- # HTTP # -------------------------------------------------------------------------- +def user_agent(cfg): + return f"batocera-store/{VERSION} ({cfg['store']['id']})" + + def _opener(cfg): - if cfg.get("insecure"): + if cfg["behavior"].get("insecure"): import ssl ctx = ssl.create_default_context() @@ -147,20 +230,20 @@ def _opener(cfg): def http_get(cfg, url): """Return (body_bytes, content_type).""" - req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - with _opener(cfg).open(req, timeout=cfg["timeout"]) as resp: + req = urllib.request.Request(url, headers={"User-Agent": user_agent(cfg)}) + with _opener(cfg).open(req, timeout=cfg["behavior"]["timeout"]) as resp: return resp.read(), resp.headers.get("Content-Type", "") def http_download(cfg, url, dest): """Stream `url` into `dest` atomically. Returns bytes written.""" - req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + req = urllib.request.Request(url, headers={"User-Agent": user_agent(cfg)}) directory = os.path.dirname(dest) or "." os.makedirs(directory, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=directory, prefix=".ttg-", suffix=".part") + fd, tmp = tempfile.mkstemp(dir=directory, prefix=f".{TAG}-", suffix=".part") written = 0 try: - with _opener(cfg).open(req, timeout=cfg["timeout"]) as resp, os.fdopen(fd, "wb") as out: + with _opener(cfg).open(req, timeout=cfg["behavior"]["timeout"]) as resp, os.fdopen(fd, "wb") as out: while True: chunk = resp.read(64 * 1024) if not chunk: @@ -183,9 +266,8 @@ def http_download(cfg, url, dest): def fetch_catalog(cfg, use_cache=False): - url = f"{cfg['base_url']}/api/software" - if cfg.get("owner_id") is not None: - url += "?" + urllib.parse.urlencode({"owner_id": cfg["owner_id"]}) + owner = cfg["catalog"].get("owner_id") + url = api_url(cfg, "catalog", {"owner_id": owner} if owner is not None else None) try: body, _ = http_get(cfg, url) data = json.loads(body.decode("utf-8")) @@ -237,9 +319,9 @@ def pick_release(entry, kind, ext): def select_games(cfg, catalog, available_systems): """Catalog -> list of installable game dicts, plus a list of skip reasons.""" - statuses = {s.lower() for s in cfg.get("statuses") or []} - only = {n.lower() for n in cfg.get("only") or []} - exclude = {n.lower() for n in cfg.get("exclude") or []} + statuses = {s.lower() for s in cfg["catalog"].get("statuses") or []} + only = {n.lower() for n in cfg["catalog"].get("only") or []} + exclude = {n.lower() for n in cfg["catalog"].get("exclude") or []} games, skipped = [], [] for entry in catalog.get("softwares") or []: @@ -297,23 +379,23 @@ def es_date(iso): def system_dir(cfg, system): - return os.path.join(cfg["roms_root"], system) + return os.path.join(roms_root(cfg), system) def detect_systems(cfg): """Systems this box has a ROM folder for — our compatibility check.""" - root = cfg["roms_root"] + root = roms_root(cfg) if not os.path.isdir(root): return set() return {d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))} def rel_rom_path(cfg, game): - return f"{cfg['subfolder']}/{game['asset']}" + return f"{subfolder(cfg)}/{game['asset']}" def rel_image_path(cfg, game, ext): - return f"{cfg['subfolder']}/images/{game['name']}{ext}" + return f"{subfolder(cfg)}/images/{game['name']}{ext}" def install_game(cfg, game, state_entry, dry_run=False): @@ -326,7 +408,7 @@ def install_game(cfg, game, state_entry, dry_run=False): if os.path.isfile(rom_abs) and os.path.getsize(rom_abs) > 0: debug(f"{game['name']}: {game['asset']} already present") else: - url = f"{cfg['base_url']}/api/download?" + urllib.parse.urlencode({"path": game["asset"]}) + url = api_url(cfg, "download", {"path": game["asset"]}) if dry_run: log(f"would download {game['name']} {game['version']} -> {rom_abs}") else: @@ -348,7 +430,7 @@ def install_game(cfg, game, state_entry, dry_run=False): # Box art disappeared from the catalog: drop the file we cached. if image_rel and not dry_run: stale = os.path.join(base, image_rel) - if image_rel.startswith(cfg["subfolder"] + "/") and os.path.isfile(stale): + if image_rel.startswith(subfolder(cfg) + "/") and os.path.isfile(stale): os.unlink(stale) changed = True image_rel = None @@ -359,10 +441,15 @@ def install_game(cfg, game, state_entry, dry_run=False): return record, changed +def image_url(cfg, game): + """The catalog gives a server-relative `imageUrl`; absolute ones pass through.""" + url = game["image_url"] + return url if url.startswith(("http://", "https://")) else base_url(cfg) + url + + def fetch_image(cfg, game, base): - url = cfg["base_url"] + game["image_url"] try: - body, ctype = http_get(cfg, url) + body, ctype = http_get(cfg, image_url(cfg, game)) except (urllib.error.URLError, OSError) as exc: log(f"warning: box art for {game['name']} failed: {exc}") return None @@ -381,9 +468,9 @@ def remove_game(cfg, record, dry_run=False): if not rel: continue path = os.path.join(base, rel) - # Never step outside our own subfolder. - if not rel.startswith(cfg["subfolder"] + "/"): - log(f"warning: refusing to delete {path} (outside {cfg['subfolder']}/)") + # Never step outside our own subfolder — another store may own it. + if not rel.startswith(subfolder(cfg) + "/"): + log(f"warning: refusing to delete {path} (outside {subfolder(cfg)}/)") continue if os.path.isfile(path): if dry_run: @@ -411,10 +498,11 @@ def merge_gamelist(cfg, system, records, dry_run=False): """Rewrite only the / nodes under our subfolder. Everything else in the file — the user's own scraped ROMs, their play - counts, favourites — is parsed and written back untouched. + counts, favourites, and whatever another store installed under its own + subfolder — is parsed and written back untouched. """ path = gamelist_path(cfg, system) - prefix = cfg["subfolder"] + "/" + prefix = subfolder(cfg) + "/" if os.path.isfile(path): try: @@ -423,7 +511,7 @@ def merge_gamelist(cfg, system, records, dry_run=False): log(f"warning: {path} is not valid XML ({exc}) — starting a fresh gamelist") root = ET.Element("gameList") else: - backup = path + ".ttg-backup" + backup = f"{path}.{cfg['store']['id']}-backup" if not os.path.exists(backup) and not dry_run: shutil.copy2(path, backup) else: @@ -431,13 +519,14 @@ def merge_gamelist(cfg, system, records, dry_run=False): for node in list(root): target = normalize_path(node.findtext("path")) - if target.startswith(prefix) or target == cfg["subfolder"]: + if target.startswith(prefix) or target == subfolder(cfg): root.remove(node) - if records and cfg.get("folder_name"): + folder_name = cfg["emulationstation"].get("folder_name") + if records and folder_name: folder = ET.SubElement(root, "folder") - ET.SubElement(folder, "path").text = "./" + cfg["subfolder"] - ET.SubElement(folder, "name").text = cfg["folder_name"] + ET.SubElement(folder, "path").text = "./" + subfolder(cfg) + ET.SubElement(folder, "name").text = folder_name for record in sorted(records, key=lambda r: r["title"].lower()): game = ET.SubElement(root, "game") @@ -528,14 +617,32 @@ def restart_es(): # -------------------------------------------------------------------------- +def game_key(record): + """Two systems may both carry a game called `foo` — key on both.""" + return f"{record['system']}:{record['name']}" + + def load_state(): state = load_json(STATE_PATH, default=None) or {} - if state.get("version") != STATE_VERSION: + version = state.get("version") + if version == 1: + state = migrate_state_v1(state) + elif version != STATE_VERSION: state = {"version": STATE_VERSION, "installed": {}} state.setdefault("installed", {}) return state +def migrate_state_v1(state): + """v1 keyed `installed` by bare software name; v2 keys by `system:name`.""" + installed = {} + for record in (state.get("installed") or {}).values(): + if record.get("system") and record.get("name"): + installed[game_key(record)] = record + log(f"migrated {len(installed)} state entries to the per-system key format") + return {"version": STATE_VERSION, "installed": installed} + + def save_state(state): state["version"] = STATE_VERSION write_json(STATE_PATH, state) @@ -548,6 +655,15 @@ def records_by_system(installed): return grouped +def match_keys(installed, names): + """Resolve user-typed `name` or `system:name` arguments to state keys.""" + wanted = {n.lower() for n in names} + return [ + key for key, record in installed.items() + if key.lower() in wanted or record["name"].lower() in wanted + ] + + # -------------------------------------------------------------------------- # commands # -------------------------------------------------------------------------- @@ -563,7 +679,7 @@ def perform_sync(cfg, names=None, dry_run=False): if not systems: # An unmounted or wrong ROMs root would otherwise look like "no system # is compatible any more" and prune everything we ever installed. - die(f"no system folders under {cfg['roms_root']} — is it mounted?") + die(f"no system folders under {roms_root(cfg)} — is it mounted?") catalog = fetch_catalog(cfg, use_cache=True) games, skipped = select_games(cfg, catalog, systems) @@ -572,14 +688,15 @@ def perform_sync(cfg, names=None, dry_run=False): if names: wanted = {n.lower() for n in names} - games = [g for g in games if g["name"].lower() in wanted] + games = [g for g in games if g["name"].lower() in wanted or game_key(g).lower() in wanted] state = load_state() installed = state["installed"] touched, changed = set(), False for game in games: - previous = installed.get(game["name"]) + key = game_key(game) + previous = installed.get(key) if previous and previous.get("asset") != game["asset"]: log(f"{game['name']}: {previous.get('version')} -> {game['version']}, removing old files") remove_game(cfg, previous, dry_run=dry_run) @@ -590,21 +707,21 @@ def perform_sync(cfg, names=None, dry_run=False): except (urllib.error.URLError, OSError) as exc: log(f"warning: {game['name']} failed: {exc}") continue - if did or installed.get(game["name"]) != record: + if did or installed.get(key) != record: changed = True - installed[game["name"]] = record + installed[key] = record touched.add(record["system"]) # A name-limited sync is not a full picture of the catalog, so never prune. - if cfg.get("prune") and not names: - keep = {g["name"] for g in games} - for name in list(installed): - if name in keep: + if cfg["behavior"].get("prune") and not names: + keep = {game_key(g) for g in games} + for key in list(installed): + if key in keep: continue - log(f"pruning {name} (no longer in the catalog or filtered out)") - remove_game(cfg, installed[name], dry_run=dry_run) - touched.add(installed[name]["system"]) - del installed[name] + log(f"pruning {key} (no longer in the catalog or filtered out)") + remove_game(cfg, installed[key], dry_run=dry_run) + touched.add(installed[key]["system"]) + del installed[key] changed = True if not dry_run: @@ -625,7 +742,7 @@ def cmd_sync(cfg, args): for system in sorted(touched | set(grouped)): merge_gamelist(cfg, system, grouped.get(system, [])) - if cfg.get("restart_es") and not args.no_restart and es_pid(): + if cfg["emulationstation"].get("restart") and not args.no_restart and es_pid(): restart_es() return 0 @@ -651,10 +768,10 @@ def cmd_sync_from_es(cfg, args): subprocess.Popen( [sys.executable, os.path.abspath(__file__), - "--config", args.config, "apply-gamelists", "--wait-pid", str(pid)], + "--config", CONFIG_PATH, "apply-gamelists", "--wait-pid", str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) - if cfg.get("restart_es") and not args.no_restart: + if cfg["emulationstation"].get("restart") and not args.no_restart: restart_es() return 0 @@ -682,7 +799,7 @@ def cmd_list(cfg, args): if not games: log("no compatible games in the catalog") for game in sorted(games, key=lambda g: (g["system"], g["title"].lower())): - record = installed.get(game["name"]) + record = installed.get(game_key(game)) if not record: mark = " " elif record.get("asset") == game["asset"]: @@ -701,15 +818,16 @@ def cmd_remove(cfg, args): state = load_state() installed = state["installed"] touched = set() - for name in args.name: - record = installed.get(name) - if not record: - log(f"{name} is not installed") - continue + keys = match_keys(installed, args.name) + if not keys: + log(f"not installed: {', '.join(args.name)}") + return 0 + for key in keys: + record = installed[key] remove_game(cfg, record, dry_run=args.dry_run) touched.add(record["system"]) if not args.dry_run: - del installed[name] + del installed[key] if args.dry_run: return 0 save_state(state) @@ -737,10 +855,11 @@ def main(argv=None): global VERBOSE parser = argparse.ArgumentParser( - prog="ttg-store", - description="Install Teletype Games catalog releases into Batocera.", + prog=os.path.basename(sys.argv[0]) or "store.py", + description="Install WarpEngine catalog releases into Batocera.", ) - parser.add_argument("--config", default=CONFIG_PATH, help=f"config file (default: {CONFIG_PATH})") + parser.add_argument("--config", default=CONFIG_PATH, + help=f"store config file (default: {CONFIG_PATH})") parser.add_argument("--base-url", help="override the catalog base URL") parser.add_argument("--roms-root", help="override the ROMs root (default: /userdata/roms)") parser.add_argument("-n", "--dry-run", action="store_true", help="report actions, change nothing") @@ -765,12 +884,12 @@ def main(argv=None): p_list = sub.add_parser("list", help="show compatible catalog entries") p_list.set_defaults(func=cmd_list) - p_rm = sub.add_parser("remove", help="uninstall games") + p_rm = sub.add_parser("remove", help="uninstall games (name or system:name)") p_rm.add_argument("name", nargs="+") p_rm.set_defaults(func=cmd_remove) p_cfg = sub.add_parser("config", help="print the effective config") - p_cfg.add_argument("--write", action="store_true", help="write the default config file") + p_cfg.add_argument("--write", action="store_true", help="write a template config file") p_cfg.add_argument("--force", action="store_true") p_cfg.set_defaults(func=cmd_config) @@ -780,7 +899,8 @@ def main(argv=None): return 2 VERBOSE = args.verbose - cfg = load_config(args.config, {"base_url": args.base_url, "roms_root": args.roms_root}) + set_home(args.config) + cfg = load_config(args.config, {"store.base_url": args.base_url, "paths.roms_root": args.roms_root}) os.makedirs(HOME, exist_ok=True) return args.func(cfg, args)