From 03a51a5b3067fd5cd1cdfc5d1fabaf894c118c77 Mon Sep 17 00:00:00 2001 From: Zsolt Tasnadi Date: Mon, 10 Aug 2026 11:16:17 +0200 Subject: [PATCH] warp_engine 0.2.0: pluggable storage adapter and publish notifications Two seams the hosts needed, both backward compatible. Storage: artifacts are served through WarpEngine::Storage.adapter instead of raw filesystem calls. The default :local adapter keeps the previous behaviour byte for byte, including the path traversal guard. A host can now set config.storage_adapter to any object answering file?/directory?/locate and serve builds from an object store - FileService and /api/download both honour a Location.redirect, so a signing adapter turns them into redirects. DownloadService#create still returns an absolute path (nil when missing) for existing callers; #locate is the new entry point that can also return a redirect. Ingestion (upload, extraction, file manager) stays local for now. Publish: PublishService emits ActiveSupport::Notifications ("warp_engine.publish") with platform/name/version/software/release, so hosts can react to a new build without hanging callbacks on the models. WarpEngine.instruments_publish? lets a host feature-detect and keep its fallback for older engine versions. --- .gitignore | 1 + README.md | 62 ++++++++++++ .../warp_engine/api/downloads_controller.rb | 11 ++- app/services/warp_engine/download_service.rb | 41 +++++--- app/services/warp_engine/file_service.rb | 31 +++--- app/services/warp_engine/publish_service.rb | 22 ++++- lib/warp_engine.rb | 13 +++ lib/warp_engine/configuration.rb | 10 +- lib/warp_engine/storage.rb | 92 +++++++++++++++++ lib/warp_engine/version.rb | 2 +- spec/lib/storage_spec.rb | 59 +++++++++++ spec/services/publish_service_spec.rb | 33 +++++++ spec/services/storage_serving_spec.rb | 98 +++++++++++++++++++ 13 files changed, 442 insertions(+), 33 deletions(-) create mode 100644 lib/warp_engine/storage.rb create mode 100644 spec/lib/storage_spec.rb create mode 100644 spec/services/storage_serving_spec.rb diff --git a/.gitignore b/.gitignore index 119a2bf..cb99d4a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ log/ spec/dummy/log/ spec/dummy/tmp/ Gemfile.lock +.bundle/ diff --git a/README.md b/README.md index abd7a81..38b7a3d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,12 @@ Repository: `https://git.teletypegames.org/tools/warp_engine` box: TIC-80, Ebitengine, LÖVE, C64, Godot, Bevy, Phaser. Authenticated by a shared secret or by per-owner database tokens with expiry and scopes (`ApplicationToken`, managed in the admin). +- **Pluggable storage**: artifacts are served through a storage adapter + (`:local` by default); a host can serve them from an object store without + patching the engine. +- **Publish events**: every published release emits + `ActiveSupport::Notifications` (`warp_engine.publish`), so hosts can react + to new builds without model callbacks. - **Public JSON API**: catalog listing, highlighted title, per-platform build matrix, image serving, download tracking, and a static file server for web-playable builds. @@ -325,6 +331,62 @@ server's repos is an admin-only endpoint (anything less yields Woodpecker server, then log out and back in: the admin flag is written to the user record at login, a server restart alone is not enough. +## Storage + +Build artifacts are served through a storage adapter. The default is the +local filesystem under `file_container_path` — byte for byte the behaviour +the engine always had: + +```ruby +c.storage_adapter = :local # default +``` + +A host that keeps its artifacts elsewhere (an object store behind a CDN, for +example) can plug in its own object instead of patching the engine. The +contract is three methods: + +```ruby +class MyObjectStore + def file?(relative_path) = ... # true/false + def directory?(relative_path) = ... # true/false + + # Return a WarpEngine::Storage::Location: + # Location.file(absolute_path) — the engine will send_file it + # Location.redirect(url) — the engine will redirect (signed URL) + def locate(relative_path, filename: nil, expires_in: nil) = ... +end + +c.storage_adapter = MyObjectStore.new +``` + +`GET /api/download` and `GET /file/*` both go through the adapter, so a +signing adapter turns them into redirects without any further change. +`WarpEngine::DownloadService#create` still returns an absolute path (and +`nil` when there is none), so existing callers keep working; +`#locate` is the new entry point that can also hand back a redirect. + +**Serving only.** Ingestion — `POST /build/upload`, archive extraction and +the admin file manager — still writes to the local disk. A remote adapter +needs its own upload path today. + +## Publish events + +Publishing a release emits an `ActiveSupport::Notifications` event, so a host +can react to a new build without hanging a callback on the models: + +```ruby +ActiveSupport::Notifications.subscribe("warp_engine.publish") do |*, payload| + payload[:software] # WarpEngine::Software + payload[:release] # WarpEngine::Release + payload[:platform] # "godot" + payload[:name] # "mygame" + payload[:version] # "1.2.0" +end +``` + +Hosts that must support older engine versions can feature-detect with +`WarpEngine.respond_to?(:instruments_publish?) && WarpEngine.instruments_publish?`. + ## Public API | Endpoint | Purpose | diff --git a/app/controllers/warp_engine/api/downloads_controller.rb b/app/controllers/warp_engine/api/downloads_controller.rb index 1bb070a..950169d 100644 --- a/app/controllers/warp_engine/api/downloads_controller.rb +++ b/app/controllers/warp_engine/api/downloads_controller.rb @@ -8,23 +8,26 @@ module WarpEngine api :GET, "/api/download", "Download a file by path" param :path, String, required: true, desc: "File path to download" returns code: 200, desc: "File binary data" + returns code: 302, desc: "Redirect to the storage location (non-local storage adapter)" error code: 400, desc: "Path is blank" error code: 404, desc: "File not found" def show path = params[:path] return render(json: { error: "Path is required" }, status: :bad_request) if path.blank? - full_path = WarpEngine::DownloadService.new.create( + location = WarpEngine::DownloadService.new.locate( path: path, ip: request.remote_ip, user_agent: request.user_agent, referer: request.referer ) - if full_path - send_file full_path, disposition: "attachment", type: resolve_mime(full_path) - else + if location.nil? render json: { error: "Not found" }, status: :not_found + elsif location.redirect? + redirect_to location.url, allow_other_host: true + else + send_file location.path, disposition: "attachment", type: resolve_mime(location.path) end end end diff --git a/app/services/warp_engine/download_service.rb b/app/services/warp_engine/download_service.rb index f21f473..cfcfb2c 100644 --- a/app/services/warp_engine/download_service.rb +++ b/app/services/warp_engine/download_service.rb @@ -9,26 +9,45 @@ module WarpEngine Pathname.new(container_base).realpath end - def create(path:, ip:, user_agent:, referer:) - sanitized = path.to_s - base_path = self.class.base_path - full_path = base_path.join(sanitized).realpath - return nil unless full_path.to_s.start_with?(base_path.to_s) - return nil unless File.file?(full_path) + # A letöltés helyét adja vissza (fájl vagy aláírt URL) és naplózza a + # letöltést. A hely feloldása a storage adapteren megy — alapból :local, + # tehát változatlanul lemezről. + def locate(path:, ip:, user_agent:, referer:) + relative = path.to_s + return nil unless storage.file?(relative) - escaped = sanitized.gsub("%", "\\%").gsub("_", "\\_") - asset = WarpEngine::ReleaseAsset.find_by(path: File.join(self.class.container_base, sanitized)) || + log_download(relative, ip: ip, user_agent: user_agent, referer: referer) + + storage.locate(relative, filename: File.basename(relative)) + end + + # Visszafelé kompatibilis felület: az abszolút fájlútvonalat adja vissza + # (vagy nil-t). Nem lemezes adapternél nincs útvonal — ott a #locate való. + def create(path:, ip:, user_agent:, referer:) + location = locate(path: path, ip: ip, user_agent: user_agent, referer: referer) + return nil if location.nil? + + location.file? ? location.path : nil + end + + private + + def storage + WarpEngine.storage + end + + def log_download(relative, ip:, user_agent:, referer:) + escaped = relative.gsub("%", "\\%").gsub("_", "\\_") + asset = WarpEngine::ReleaseAsset.find_by(path: File.join(self.class.container_base, relative)) || WarpEngine::ReleaseAsset.where("path LIKE ?", "%#{escaped}%").first WarpEngine::Download.create!( - file_path: sanitized, + file_path: relative, release: asset&.release, ip_address: ip, user_agent: user_agent&.truncate(500), referer: referer&.truncate(500) ) - - full_path end end end diff --git a/app/services/warp_engine/file_service.rb b/app/services/warp_engine/file_service.rb index d50191e..2198c2c 100644 --- a/app/services/warp_engine/file_service.rb +++ b/app/services/warp_engine/file_service.rb @@ -5,31 +5,32 @@ module WarpEngine Pathname.new(WarpEngine.config.file_container_path).realpath end + # A fájlok helyét a storage adapter adja (alapból :local, azaz a lemez) — + # így a host az objektumtárból is kiszolgálhat anélkül, hogy az engine-t + # patchelné. Lásd WarpEngine::Storage. def show(input) - full_path = base_path.join(input.path.to_s) - return FileResultDto.not_found unless safe_path?(full_path) + relative = input.path.to_s - if File.directory?(full_path) - index_path = full_path.join("index.html") - return FileResultDto.not_found unless File.file?(index_path) - return FileResultDto.redirect("/file/#{input.path.to_s.chomp("/")}/index.html") + if storage.directory?(relative) + index = File.join(relative.chomp("/"), "index.html") + return FileResultDto.not_found unless storage.file?(index) + + return FileResultDto.redirect("/file/#{index}") end - if File.file?(full_path) - FileResultDto.file(full_path) - else - FileResultDto.not_found - end + return FileResultDto.not_found unless storage.file?(relative) + + to_result(storage.locate(relative, filename: File.basename(relative))) end private - def base_path - @base_path ||= self.class.base_path + def storage + WarpEngine.storage end - def safe_path?(path) - File.exist?(path) && Pathname.new(path).realpath.to_s.start_with?(base_path.to_s) + def to_result(location) + location.redirect? ? FileResultDto.redirect(location.url) : FileResultDto.file(location.path) end end end diff --git a/app/services/warp_engine/publish_service.rb b/app/services/warp_engine/publish_service.rb index d864647..c5eb635 100644 --- a/app/services/warp_engine/publish_service.rb +++ b/app/services/warp_engine/publish_service.rb @@ -1,11 +1,31 @@ module WarpEngine class PublishService + # Az esemény neve, amit a host lehallgathat. A payload: + # platform: String, name: String, version: String, + # software: WarpEngine::Software, release: WarpEngine::Release + NOTIFICATION = "warp_engine.publish".freeze + def publish(input) unless WarpEngine::PlatformLink::SUPPORTED_PLATFORMS.include?(input.platform) raise ArgumentError, "Unsupported platform: #{input.platform}" end - "WarpEngine::Platforms::#{input.platform.camelize}::Service".constantize.new.update(input.name, input.version) + release = "WarpEngine::Platforms::#{input.platform.camelize}::Service".constantize + .new.update(input.name, input.version) + + # A publikálás az egyetlen pont, ahol új build kerül a katalógusba — + # a host innen tud rá reagálni (értesítés, feed, csatorna-előléptetés) + # anélkül, hogy modell-callbackre kellene kapaszkodnia. + ActiveSupport::Notifications.instrument( + NOTIFICATION, + platform: input.platform, + name: input.name, + version: input.version, + software: release.respond_to?(:software) ? release.software : nil, + release: release + ) + + release end end end diff --git a/lib/warp_engine.rb b/lib/warp_engine.rb index 1bc258d..0a4ceee 100644 --- a/lib/warp_engine.rb +++ b/lib/warp_engine.rb @@ -3,6 +3,7 @@ require "apipie-rails" require "warp_engine/version" require "warp_engine/configuration" +require "warp_engine/storage" module WarpEngine # A tábláink prefix nélküliek (softwares, releases, ...) — az isolate_namespace @@ -23,6 +24,18 @@ module WarpEngine def self.woodpecker_configured? config.woodpecker_url.present? && config.woodpecker_api_token.present? end + + # A host innen tudja, hogy a publikálás ActiveSupport::Notifications-t szór + # ("warp_engine.publish"), és nem kell modell-callbackre kapaszkodnia. + # Régebbi engine-verziókon a metódus nem létezik, ezért a hívó oldalon + # respond_to?-val kérdezendő. + def self.instruments_publish? + true + end + + def self.storage + Storage.adapter + end end require "warp_engine/engine" diff --git a/lib/warp_engine/configuration.rb b/lib/warp_engine/configuration.rb index 2b2e8bc..502782c 100644 --- a/lib/warp_engine/configuration.rb +++ b/lib/warp_engine/configuration.rb @@ -9,6 +9,12 @@ module WarpEngine # :database — only WarpEngine::ApplicationToken is accepted, the shared secret is not # application_token_owner_class: class name of the mandatory token owner # (e.g. "AdminUser"); nil makes :database mode reject every request. + # storage_adapter: where build artifacts are served from. + # :local (default) — the local filesystem under file_container_path, + # byte for byte the previous behaviour; + # any object — must answer file?/directory?/locate, see + # WarpEngine::Storage. Serving only: uploads and + # archive extraction still write to the local disk. # max_upload_size: file size cap in bytes for /build/upload (and the admin file manager). # enforce_software_ownership: when true, a DB token may only upload/publish # its own owner's softwares (unrestricted tokens are exempt). @@ -37,7 +43,8 @@ module WarpEngine :woodpecker_url, :woodpecker_api_token, :woodpecker_repo_owner, - :image_owners + :image_owners, + :storage_adapter def initialize @file_container_path = ENV.fetch("FILE_CONTAINER_PATH", "/softwares") @@ -55,6 +62,7 @@ module WarpEngine @woodpecker_api_token = ENV["WOODPECKER_API_TOKEN"] @woodpecker_repo_owner = ENV["WOODPECKER_REPO_OWNER"] @image_owners = [] + @storage_adapter = :local end end end diff --git a/lib/warp_engine/storage.rb b/lib/warp_engine/storage.rb new file mode 100644 index 0000000..f8f889c --- /dev/null +++ b/lib/warp_engine/storage.rb @@ -0,0 +1,92 @@ +module WarpEngine + # Where the artifacts physically live. + # + # Until now every path in the engine was a local filesystem path. This module + # is the seam a host needs to serve builds from somewhere else (an object + # store behind a CDN, for example) without patching the engine. + # + # The default adapter is :local and behaves exactly as before - same paths, + # same traversal protection, same File.file? checks. + # + # A custom adapter is any object answering to this contract: + # + # file?(relative_path) -> true/false + # directory?(relative_path) -> true/false + # locate(relative_path, filename: nil, expires_in: nil) -> Location + # + # It is set on the configuration: + # + # c.storage_adapter = MyObjectStore.new # or :local (default) + # + # NOTE: ingestion (build/upload, archive extraction, the admin file manager) + # still writes to the local filesystem. A remote adapter therefore needs its + # own upload path today; the serving side is what this seam covers. + module Storage + Location = Struct.new(:kind, :path, :url, keyword_init: true) do + def file? = kind == :file + def redirect? = kind == :redirect + + def self.file(path) = new(kind: :file, path: path) + def self.redirect(url) = new(kind: :redirect, url: url) + end + + # The local filesystem, rooted at config.file_container_path. + class LocalAdapter + def base_path + Pathname.new(WarpEngine.config.file_container_path) + end + + def absolute_path(relative_path) + base_path.join(relative_path.to_s) + end + + def file?(relative_path) + path = absolute_path(relative_path) + File.file?(path) && inside_base?(path) + end + + def directory?(relative_path) + path = absolute_path(relative_path) + File.directory?(path) && inside_base?(path) + end + + # expires_in is part of the contract for signing adapters; the local + # filesystem has nothing to sign, so it is ignored here. + def locate(relative_path, filename: nil, expires_in: nil) + Location.file(absolute_path(relative_path).to_s) + end + + private + + # Path traversal guard: the resolved path must stay under the container. + def inside_base?(path) + root = base_path.realpath.to_s + Pathname.new(path).realpath.to_s.start_with?(root) + rescue Errno::ENOENT + false + end + end + + class << self + def adapter + configured = WarpEngine.config.storage_adapter + + case configured + when nil, :local, "local" then local_adapter + else configured + end + end + + def local? = adapter.is_a?(LocalAdapter) + + def local_adapter + @local_adapter ||= LocalAdapter.new + end + + # Tests and hosts that swap the configuration at runtime. + def reset! + @local_adapter = nil + end + end + end +end diff --git a/lib/warp_engine/version.rb b/lib/warp_engine/version.rb index 1ebbfd3..7349cc2 100644 --- a/lib/warp_engine/version.rb +++ b/lib/warp_engine/version.rb @@ -1,3 +1,3 @@ module WarpEngine - VERSION = "0.1.0" + VERSION = "0.2.0" end diff --git a/spec/lib/storage_spec.rb b/spec/lib/storage_spec.rb new file mode 100644 index 0000000..e4c8e3d --- /dev/null +++ b/spec/lib/storage_spec.rb @@ -0,0 +1,59 @@ +require "rails_helper" +require "tmpdir" + +RSpec.describe WarpEngine::Storage do + let(:tmpdir) { Dir.mktmpdir } + + before do + allow(WarpEngine.config).to receive(:file_container_path).and_return(tmpdir) + WarpEngine::Storage.reset! + end + + after { FileUtils.rm_rf(tmpdir) } + + describe ".adapter" do + it "defaults to the local filesystem" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(:local) + + expect(described_class.adapter).to be_a(described_class::LocalAdapter) + expect(described_class).to be_local + end + + it "returns whatever object the host configured" do + custom = Object.new + allow(WarpEngine.config).to receive(:storage_adapter).and_return(custom) + + expect(described_class.adapter).to eq(custom) + expect(described_class).not_to be_local + end + end + + describe described_class::LocalAdapter do + subject(:adapter) { described_class.new } + + before { File.write(File.join(tmpdir, "game-1.0.zip"), "zip") } + + it "sees files and directories under the container" do + FileUtils.mkdir_p(File.join(tmpdir, "game-1.0")) + + expect(adapter.file?("game-1.0.zip")).to be(true) + expect(adapter.directory?("game-1.0")).to be(true) + expect(adapter.file?("missing.zip")).to be(false) + end + + # The path traversal guard has to survive the move behind the adapter. + it "refuses paths escaping the container" do + outside = File.join(Dir.mktmpdir, "secret.txt") + File.write(outside, "nope") + + expect(adapter.file?("../#{File.basename(File.dirname(outside))}/secret.txt")).to be(false) + end + + it "locates a file as an absolute path" do + location = adapter.locate("game-1.0.zip") + + expect(location).to be_file + expect(location.path).to eq(File.join(tmpdir, "game-1.0.zip")) + end + end +end diff --git a/spec/services/publish_service_spec.rb b/spec/services/publish_service_spec.rb index 6b6b564..b79a02b 100644 --- a/spec/services/publish_service_spec.rb +++ b/spec/services/publish_service_spec.rb @@ -55,5 +55,38 @@ RSpec.describe WarpEngine::PublishService do expect(mock_service).to have_received(:update).with("game", "1.0") end + describe "instrumentation" do + let(:input) { WarpEngine::PublishInputDto.new(platform: "tic80", name: "game", version: "1.0") } + let(:release) { WarpEngine::Release.new(version: "1.0") } + + before do + mock_service = instance_double(WarpEngine::Platforms::Tic80::Service, update: release) + allow(WarpEngine::Platforms::Tic80::Service).to receive(:new).and_return(mock_service) + end + + # The host reacts to a new build through this event instead of hanging a + # callback on the Release model. + it "emits warp_engine.publish with the release in the payload" do + payloads = [] + ActiveSupport::Notifications.subscribe(described_class::NOTIFICATION) do |*, payload| + payloads << payload + end + + described_class.new.publish(input) + + expect(payloads.size).to eq(1) + expect(payloads.first).to include(platform: "tic80", name: "game", version: "1.0", release: release) + ensure + ActiveSupport::Notifications.unsubscribe(described_class::NOTIFICATION) + end + + it "returns the release" do + expect(described_class.new.publish(input)).to eq(release) + end + + it "advertises the feature so older hosts can feature-detect" do + expect(WarpEngine.instruments_publish?).to be(true) + end + end end end diff --git a/spec/services/storage_serving_spec.rb b/spec/services/storage_serving_spec.rb new file mode 100644 index 0000000..b62182c --- /dev/null +++ b/spec/services/storage_serving_spec.rb @@ -0,0 +1,98 @@ +require "rails_helper" +require "tmpdir" + +# The serving side (FileService, DownloadService) goes through the storage +# adapter. With :local nothing changes; with a custom adapter the same call +# can hand back a redirect instead of a file. +RSpec.describe "Serving through the storage adapter" do + let(:tmpdir) { Dir.mktmpdir } + + # Minimal adapter answering the documented contract. + let(:signing_adapter) do + Class.new do + def file?(_relative) = true + def directory?(_relative) = false + + def locate(relative, filename: nil, expires_in: nil) + WarpEngine::Storage::Location.redirect("https://cdn.example/#{relative}?signed=1") + end + end.new + end + + before do + allow(WarpEngine.config).to receive(:file_container_path).and_return(tmpdir) + WarpEngine::Storage.reset! + File.write(File.join(tmpdir, "game-1.0.zip"), "zip") + end + + after { FileUtils.rm_rf(tmpdir) } + + describe WarpEngine::FileService do + it "serves a local file (default adapter)" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(:local) + + result = described_class.new.show(WarpEngine::FileShowInputDto.new(path: "game-1.0.zip")) + + expect(result.type).to eq(:file) + expect(result.path).to eq(File.join(tmpdir, "game-1.0.zip")) + end + + it "redirects a directory to its index.html" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(:local) + FileUtils.mkdir_p(File.join(tmpdir, "game-1.0")) + File.write(File.join(tmpdir, "game-1.0", "index.html"), "

hi

") + + result = described_class.new.show(WarpEngine::FileShowInputDto.new(path: "game-1.0")) + + expect(result.type).to eq(:redirect) + expect(result.url).to eq("/file/game-1.0/index.html") + end + + it "hands back the adapter's redirect when storage is remote" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(signing_adapter) + + result = described_class.new.show(WarpEngine::FileShowInputDto.new(path: "game-1.0.zip")) + + expect(result.type).to eq(:redirect) + expect(result.url).to eq("https://cdn.example/game-1.0.zip?signed=1") + end + + it "reports a missing file" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(:local) + + result = described_class.new.show(WarpEngine::FileShowInputDto.new(path: "nope.zip")) + + expect(result.type).to eq(:not_found) + end + end + + describe WarpEngine::DownloadService do + let(:args) { { ip: "127.0.0.1", user_agent: "rspec", referer: nil } } + + it "keeps returning an absolute path from #create (backward compatible)" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(:local) + + path = described_class.new.create(path: "game-1.0.zip", **args) + + expect(path).to eq(File.join(tmpdir, "game-1.0.zip")) + expect(WarpEngine::Download.count).to eq(1) + end + + it "returns nil from #create for a missing file" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(:local) + + expect(described_class.new.create(path: "nope.zip", **args)).to be_nil + expect(WarpEngine::Download.count).to eq(0) + end + + it "returns a location from #locate and logs the download" do + allow(WarpEngine.config).to receive(:storage_adapter).and_return(signing_adapter) + + location = described_class.new.locate(path: "game-1.0.zip", **args) + + expect(location).to be_redirect + expect(location.url).to include("signed=1") + expect(WarpEngine::Download.count).to eq(1) + end + end +end