Refuse loads and renders that would not fit in memory

Opening a project of ninety 17 MP photos got Effector killed by the
kernel's OOM killer: 4.6 GB of decoded RGB on a machine with 3.6 GB free,
and no core dump to show for it. Bulk allocations are now sized first and
refused with an OutOfMemoryError past three quarters of the free memory
(MemAvailable, or less under a cgroup memory.max). loadImages sizes the
batch from the file headers and refuses it before decoding anything, and
holds the decodes in flight to what is left; renderChain and renderFrames
refuse a canvas at 24 bytes a pixel, plus any picture a Path param loads.
The interface keeps a refused batch's paths, so saving the project never
drops them. Version 0.6.1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/core/chain.cpp

@@ -2,6 +2,8 @@
 
 #include <algorithm>
 
+#include "core/memory.hpp"
+
 namespace effector {
 
 std::vector<const ChainStage*> enabledStages(const std::vector<ChainStage>& stages) {
@@ -48,6 +50,28 @@ std::vector<const ChainStage*> runningStages(const std::vector<ChainStage>& stag
     return running;
 }
 
+// Throws OutOfMemoryError for a render that would not fit. A render holds a
+// few canvases at once -- the stage before, the one being drawn, the input
+// a transform copies -- and the file it is saved to; 13 bytes a pixel is
+// what the heaviest chains were measured at, and the interface keeps one
+// more copy to show, so 24 leaves room. A picture a stage loads itself (an
+// overlay, a text box's picture) comes on top, with its alpha and pixbuf.
+void ensureRenderFits(const std::vector<const ChainStage*>& running, int width, int height,
+                      const StyleSet& styles) {
+    std::size_t needed = static_cast<std::size_t>(std::max(width, 0)) *
+                         static_cast<std::size_t>(std::max(height, 0)) * 24;
+    for (const auto* stage : running) {
+        const auto values = stageValues(*stage, styles);
+        for (const auto& spec : stage->effect->params()) {
+            if (spec.type != ParamSpec::Type::Path) continue;
+            const std::string path = paramText(values, spec.key, "");
+            if (!path.empty()) needed += imageFilePixels(path) * 8;
+        }
+    }
+    ensureMemory(needed, "Rendering a " + std::to_string(width) + " × " + std::to_string(height) +
+                             " canvas");
+}
+
 // Runs running[first, last) on `image`, which holds the sources' place for
 // the first stage of the chain and the previous stage's output after it.
 Image runStages(const std::vector<const ChainStage*>& running, std::size_t first,
@@ -76,6 +100,7 @@ Image renderChain(const std::vector<ChainStage>& stages, const std::vector<Image
                   int width, int height, Rng& rng, const StyleSet& styles,
                   const FrameTime& frame) {
     const auto running = runningStages(stages);
+    ensureRenderFits(running, width, height, styles);
     return runStages(running, 0, running.size(), sources, Image(), width, height, rng, styles,
                      frame);
 }
@@ -92,6 +117,7 @@ int renderFrames(const std::vector<ChainStage>& stages, const std::vector<Image>
                  const VideoSettings& video,
                  const std::function<bool(int index, Image&& frame)>& onFrame) {
     const auto running = runningStages(stages);
+    ensureRenderFits(running, width, height, styles);
     const int count = video.frameCount();
     const int fps = std::clamp(video.fps, kMinFps, kMaxFps);
 

src/core/image.cpp

@@ -11,6 +11,8 @@
 #include <stdexcept>
 #include <thread>
 
+#include "core/memory.hpp"
+
 namespace effector {
 namespace {
 
@@ -152,9 +154,31 @@ void Image::save(const std::string& path) const {
     if (saved == FALSE) throw std::runtime_error(takeError(error));
 }
 
+std::size_t imageFilePixels(const std::string& path) {
+    int width = 0;
+    int height = 0;
+    if (gdk_pixbuf_get_file_info(path.c_str(), &width, &height) == nullptr) return 0;
+    return static_cast<std::size_t>(std::max(width, 0)) * static_cast<std::size_t>(std::max(height, 0));
+}
+
 std::vector<Image> loadImages(const std::vector<std::string>& paths,
                               std::vector<LoadFailure>& failures,
                               const LoadProgress* progress) {
+    // What the batch holds once decoded, and what one decode needs on top
+    // while it runs: the pixbuf (with alpha, at worst) and its rotated copy.
+    std::size_t kept = 0;
+    std::size_t largest = 0;
+    for (const auto& path : paths) {
+        const std::size_t pixels = imageFilePixels(path);
+        kept += pixels * 3;
+        largest = std::max(largest, pixels * 8);
+    }
+    const std::string what = paths.size() == 1
+                                 ? std::string("Loading this image")
+                                 : "Loading these " + std::to_string(paths.size()) + " images";
+    ensureMemory(kept + largest, what);
+    const std::size_t spare = usableMemory() - std::min(usableMemory(), kept);
+
     // Decoding is the dominant cost of a headless run, and the files are
     // independent, so they are decoded in parallel. Threads take the next path
     // from a shared cursor rather than a fixed slice, so one huge file cannot
@@ -167,7 +191,8 @@ std::vector<Image> loadImages(const std::vector<std::string>& paths,
 
     unsigned hardware = std::thread::hardware_concurrency();
     if (hardware == 0) hardware = 1;
-    const std::size_t workers = std::min<std::size_t>(hardware, std::max<std::size_t>(paths.size(), 1));
+    std::size_t workers = std::min<std::size_t>(hardware, std::max<std::size_t>(paths.size(), 1));
+    if (largest > 0) workers = std::clamp<std::size_t>(spare / largest, 1, workers);
 
     const auto decode = [&] {
         for (;;) {

src/core/image.hpp

@@ -68,9 +68,18 @@ struct LoadProgress {
     std::atomic<bool>* cancel = nullptr;
 };
 
+// The pixel count of an image file from its header alone, without decoding
+// it; 0 when the header cannot be read.
+std::size_t imageFilePixels(const std::string& path);
+
 // Loads every path, collecting rather than throwing on individual failures.
 // When `progress->cancel` is set the remaining paths are skipped, so the result
 // is short and should be discarded rather than used.
+//
+// Before decoding anything the whole batch is sized from the file headers,
+// and a batch that would not fit in the free memory throws OutOfMemoryError
+// (core/memory.hpp) with nothing loaded; the decodes that run at once are
+// also held to what is left over.
 std::vector<Image> loadImages(const std::vector<std::string>& paths,
                               std::vector<LoadFailure>& failures,
                               const LoadProgress* progress = nullptr);

src/core/memory.cpp

@@ -0,0 +1,123 @@
+#include "core/memory.hpp"
+
+#include <unistd.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+
+namespace effector {
+namespace {
+
+std::atomic<std::size_t> cap{0};
+
+// MemAvailable from /proc/meminfo, falling back on the free pages; 0 when
+// neither says.
+std::size_t systemAvailable() {
+    std::ifstream meminfo("/proc/meminfo");
+    std::string key;
+    std::size_t kilobytes = 0;
+    std::string unit;
+    while (meminfo >> key >> kilobytes >> unit) {
+        if (key == "MemAvailable:") return kilobytes * 1024;
+    }
+    const long pages = sysconf(_SC_AVPHYS_PAGES);
+    const long pageSize = sysconf(_SC_PAGESIZE);
+    if (pages > 0 && pageSize > 0) {
+        return static_cast<std::size_t>(pages) * static_cast<std::size_t>(pageSize);
+    }
+    return 0;
+}
+
+// A number from a cgroup file, or nothing for "max" or a file that is not
+// there.
+bool readCount(const std::filesystem::path& file, std::size_t& out) {
+    std::ifstream in(file);
+    std::string word;
+    if (!(in >> word) || word == "max") return false;
+    try {
+        out = static_cast<std::size_t>(std::stoull(word));
+        return true;
+    } catch (const std::exception&) {
+        return false;
+    }
+}
+
+std::size_t inactiveFile(const std::filesystem::path& dir) {
+    std::ifstream in(dir / "memory.stat");
+    std::string key;
+    std::size_t value = 0;
+    while (in >> key >> value) {
+        if (key == "inactive_file") return value;
+    }
+    return 0;
+}
+
+// The room left under the tightest memory.max on the way from this
+// process's cgroup (v2) up to the root, counting the page cache it could
+// drop as room; the largest value when nothing caps it.
+std::size_t cgroupAvailable() {
+    std::size_t room = std::numeric_limits<std::size_t>::max();
+    std::ifstream self("/proc/self/cgroup");
+    std::string line;
+    while (std::getline(self, line)) {
+        if (line.rfind("0::", 0) != 0) continue;
+        // "0::/user.slice/app.slice/x.scope", walked from the scope upwards.
+        for (std::filesystem::path group = line.substr(4); !group.empty();
+             group = group.parent_path()) {
+            const auto dir = std::filesystem::path("/sys/fs/cgroup") / group;
+            std::size_t max = 0;
+            std::size_t current = 0;
+            if (!readCount(dir / "memory.max", max) || !readCount(dir / "memory.current", current)) {
+                continue;
+            }
+            const std::size_t used = current - std::min(current, inactiveFile(dir));
+            room = std::min(room, max - std::min(max, used));
+        }
+    }
+    return room;
+}
+
+}  // namespace
+
+std::size_t availableMemory() {
+    std::size_t available = systemAvailable();
+    const std::size_t room = cgroupAvailable();
+    if (room != std::numeric_limits<std::size_t>::max()) {
+        available = available == 0 ? room : std::min(available, room);
+    }
+    const std::size_t capped = cap.load(std::memory_order_relaxed);
+    if (capped != 0 && (available == 0 || capped < available)) available = capped;
+    return available;
+}
+
+void limitAvailableMemory(std::size_t bytes) { cap.store(bytes, std::memory_order_relaxed); }
+
+std::size_t usableMemory() {
+    const std::size_t available = availableMemory();
+    return available == 0 ? std::numeric_limits<std::size_t>::max() : available / 4 * 3;
+}
+
+void ensureMemory(std::size_t needed, const std::string& what) {
+    const std::size_t available = availableMemory();
+    if (available == 0 || needed <= available / 4 * 3) return;
+    throw OutOfMemoryError(what + " needs about " + formatBytes(needed) +
+                           " of memory, more than is safe to take with " + formatBytes(available) +
+                           " free.");
+}
+
+std::string formatBytes(std::size_t bytes) {
+    char text[32];
+    const double gb = static_cast<double>(bytes) / (1024.0 * 1024.0 * 1024.0);
+    if (gb >= 1.0) {
+        std::snprintf(text, sizeof text, "%.1f GB", gb);
+    } else {
+        std::snprintf(text, sizeof text, "%.0f MB", static_cast<double>(bytes) / (1024.0 * 1024.0));
+    }
+    return text;
+}
+
+}  // namespace effector

src/core/memory.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <cstddef>
+#include <stdexcept>
+#include <string>
+
+namespace effector {
+
+// Thrown instead of starting work that would not fit in memory. Decoding a
+// batch of photographs or rendering a huge canvas past what the machine has
+// free does not fail cleanly: the kernel's OOM killer ends the program, and
+// often others with it.
+class OutOfMemoryError : public std::runtime_error {
+public:
+    using std::runtime_error::runtime_error;
+};
+
+// The bytes this process could still allocate without swapping: the
+// system's MemAvailable, or less when a cgroup (a systemd unit, a container)
+// caps the process below it. Zero when neither can be read.
+std::size_t availableMemory();
+
+// Caps what availableMemory() reports, for the tests; 0 lifts the cap.
+void limitAvailableMemory(std::size_t bytes);
+
+// Throws OutOfMemoryError when `needed` more bytes would take more than three
+// quarters of availableMemory() -- the rest is left to the desktop and to
+// whatever else the program holds. `what` names the work in the message.
+// Does nothing when the free memory cannot be read.
+void ensureMemory(std::size_t needed, const std::string& what);
+
+// The bytes ensureMemory() would let through right now; the largest value
+// when the free memory cannot be read.
+std::size_t usableMemory();
+
+// "4.6 GB", "380 MB".
+std::string formatBytes(std::size_t bytes);
+
+}  // namespace effector

src/gui/window.cpp

@@ -10,6 +10,7 @@
 
 #include "core/canvas.hpp"
 #include "core/chain.hpp"
+#include "core/memory.hpp"
 #include "core/project.hpp"
 #include "core/registry.hpp"
 #include "core/rng.hpp"
@@ -1349,11 +1350,19 @@ void EffectorWindow::addPaths(const std::vector<std::string>& paths) {
         };
 
         std::vector<LoadFailure> failures;
-        auto images = loadImages(paths, failures, &progress);
+        std::vector<Image> images;
+        std::string refused;
+        try {
+            images = loadImages(paths, failures, &progress);
+        } catch (const OutOfMemoryError& error) {
+            refused = error.what();
+        }
         {
             std::lock_guard<std::mutex> lock(loadMutex_);
             loadedImages_ = std::move(images);
             loadFailures_ = std::move(failures);
+            loadRefused_ = std::move(refused);
+            loadRefusedPaths_ = loadRefused_.empty() ? std::vector<std::string>{} : paths;
         }
         loadFinished_.emit();
     });
@@ -1384,12 +1393,18 @@ void EffectorWindow::onLoadFinished() {
 
     std::vector<Image> images;
     std::vector<LoadFailure> failures;
+    std::string refused;
+    std::vector<std::string> refusedPaths;
     {
         std::lock_guard<std::mutex> lock(loadMutex_);
         images = std::move(loadedImages_);
         failures = std::move(loadFailures_);
+        refused = std::move(loadRefused_);
+        refusedPaths = std::move(loadRefusedPaths_);
         loadedImages_.clear();
         loadFailures_.clear();
+        loadRefused_.clear();
+        loadRefusedPaths_.clear();
     }
 
     if (loadCancel_.load(std::memory_order_relaxed)) {
@@ -1406,6 +1421,22 @@ void EffectorWindow::onLoadFinished() {
         dropLoadedImages_ = false;
         images.clear();
         failures.clear();
+        refused.clear();
+        refusedPaths.clear();
+    }
+
+    if (!refused.empty()) {
+        // Nothing of a refused batch was decoded. Its paths are kept for the
+        // project file, and whatever was queued behind it still gets its turn.
+        refusedSources_.insert(refusedSources_.end(), refusedPaths.begin(), refusedPaths.end());
+        showError(refused + " They were not loaded, but stay in the project when it is saved. "
+                            "Close other programs, or load fewer or smaller images.");
+        if (!loadQueue_.empty()) {
+            auto queued = std::move(loadQueue_);
+            loadQueue_.clear();
+            addPaths(queued);
+        }
+        return;
     }
 
     for (const auto& image : images) appendSourceRow(image.path());
@@ -2273,6 +2304,7 @@ Gtk::Widget& EffectorWindow::buildMainMenu() {
 Project EffectorWindow::currentProject() const {
     Project project;
     for (const auto& image : *sources_) project.sources.push_back(image.path());
+    project.sources.insert(project.sources.end(), refusedSources_.begin(), refusedSources_.end());
     project.stages = stages_;
     project.styles = styles_;
     project.width = static_cast<int>(widthSpin_.get_value());
@@ -2320,6 +2352,7 @@ void EffectorWindow::clearSources() {
     while (auto* row = sourceList_.get_row_at_index(0)) sourceList_.remove(*row);
     updateSourcesLabel();
     mutateSources([](SourceList& list) { list.clear(); });
+    refusedSources_.clear();
     loadQueue_.clear();
     if (loading_) dropLoadedImages_ = true;
 

src/gui/window.hpp

@@ -219,6 +219,12 @@ private:
     std::string loadCurrent_;
     std::vector<Image> loadedImages_;
     std::vector<LoadFailure> loadFailures_;
+    std::string loadRefused_;  // why the batch would not fit in memory, if it would not
+    std::vector<std::string> loadRefusedPaths_;
+    // Sources refused as too large to hold. They are not shown or rendered,
+    // but a project saved now keeps them, so a refusal never costs the
+    // project its images.
+    std::vector<std::string> refusedSources_;
     std::vector<std::string> loadQueue_;  // chosen while another batch was loading
     bool loading_ = false;
     bool dropLoadedImages_ = false;  // the batch in flight belongs to a replaced project