app/services/backup_repository/tracking.rb

Rama develop · réplica de lectura del repositorio original.

class BackupRepository
  # Shared DB-tracking layer for every adapter: listing, completion
  # detection, prune/destroy, and runner-error capture. Per-adapter IO
  # (deleting tars, reading error files, fetching tar metadata) is
  # supplied by Local/External/S3.
  #
  # Adapters must provide: `destination_key`, `destination_coords`,
  # `destination_configured?`, `remove_files!(filenames)`,
  # `record_backup!(filename)`.
  #
  # Error-file IO is destination-independent: backup.sh / restore.sh write
  # error.txt into the HELIOS-local runtime dir (see the comment in those
  # scripts), so all adapters share the same read/remove implementation
  # provided here.
  module Tracking # rubocop:disable Metrics/ModuleLength
    include Loggable

    # How often reading a freshly written tar is retried before HELIOS gives
    # up and marks the run incomplete. Guards against a backup that genuinely
    # produced no tar (host crash, no error file) being probed on every tick
    # forever, while still surviving a transient NAS/docker hiccup.
    RECORDING_MAX_ATTEMPTS = 10

    def all
      return Backup.none unless destination_configured?

      Backup.for_destination(destination_key, **destination_coords).newest_first
    end

    def latest
      all.first
    end

    def find!(filename)
      fetch_record!(filename)
    end

    def destroy!(filename)
      record = fetch_record!(filename)
      remove_files!([filename])
      record.destroy!
    end

    def prune!(keep: BackupRepository::MAX_BACKUPS)
      return unless destination_configured?

      stale = scoped_records.newest_first.offset(keep).to_a
      return if stale.empty?

      remove_files!(stale.map(&:filename))
      Backup.where(id: stale.map(&:id)).delete_all
    end

    def clear_error!(filename = BackupRepository::ERROR_FILENAME)
      remove_error_file!(filename)
      RunnerLog.clear!(RunnerLog.kind_for(filename))
    end

    def read_error_file(filename = BackupRepository::ERROR_FILENAME)
      ::File.read(runtime_error_path(filename)).strip.presence
    rescue Errno::ENOENT
      nil
    end

    def remove_error_file!(filename = BackupRepository::ERROR_FILENAME)
      FileUtils.rm_f(runtime_error_path(filename))
    end

    # Mirrors backup.sh's failure contract from the Ruby side (e.g. when the
    # BackupRunner preparing thread fails before the sidecar exists): write
    # the message into the same runtime error.txt that detect_completion!
    # later turns into a red completion card.
    def write_error_file!(message, filename = BackupRepository::ERROR_FILENAME)
      path = runtime_error_path(filename)
      FileUtils.mkdir_p(::File.dirname(path))
      ::File.write(path, message)
    end

    def runtime_error_path(filename)
      ::File.join(DetachedRunner.runtime_directory, filename)
    end

    # Detects the moment a detached BackupRunner/RestoreRunner finished
    # and persists what it left behind: error.txt → RunnerLog, fresh tar
    # → Backup. Triggered by either a live in_progress observation
    # (Rails.cache marker) or the pending-marker file from start.
    def detect_completion! # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
      cached = Rails.cache.read(in_progress_cache_key)
      pending = pending_refresh?

      # Idle fast-path. The BackupRunner/RestoreRunner probes below each shell
      # out to `docker inspect`; the scheduler calls this every 30 s, so on a
      # host with no backup activity that would be two docker subprocesses per
      # tick, all day. Both runners drop a durable pending marker before their
      # sidecar launches (BackupRunner#run_preparing!, RestoreRunner.start), so
      # any run that could need recording leaves either that marker or the
      # cached observation from a previous tick. Neither present ⇒ nothing to
      # detect, so return before touching docker.
      return unless cached || pending

      backup_ip = BackupRunner.in_progress
      restore_ip = RestoreRunner.in_progress
      current = backup_ip&.filename || restore_ip&.filename

      if current
        Rails.cache.write(in_progress_cache_key, current, expires_in: 1.hour)
        return
      end

      logger.info(
        "[detect_completion!] firing — backup_ip=#{backup_ip.inspect} restore_ip=#{restore_ip.inspect} " \
        "cached=#{cached.inspect} pending=#{pending} marker=#{pending_marker_content.inspect}",
      )
      Rails.cache.delete(in_progress_cache_key)
      process_completion!(pending_marker_content)
    end

    # The expected filename argument lets process_completion! report a
    # "ran but never produced a tar" failure even if backup.sh died
    # without writing error.txt (OOM, host crash).
    def mark_pending!(expected_filename = nil)
      return unless destination_configured?

      FileUtils.mkdir_p(::File.dirname(pending_marker_path))
      ::File.write(pending_marker_path, expected_filename.to_s)
    end

    def pending_refresh?
      ::File.exist?(pending_marker_path)
    end

    def clear_pending!
      FileUtils.rm_f(pending_marker_path)
    end

    def pending_marker_path
      ::File.join(Rails.configuration.data_path, 'helios', "#{destination_key}_backup_pending.flag")
    end

    # Filename recorded in the pending marker, or nil. Public hook used by
    # BackupRunner.in_progress to detect a tar that needs re-uploading
    # after a HELIOS restart.
    def pending_filename
      pending_marker_content
    end

    private

    def fetch_record!(filename)
      raise BackupRepository::NotFound unless BackupRepository.valid_filename?(filename)
      raise BackupRepository::NotFound unless destination_configured?

      scoped_records.find_by(filename: filename) || raise(BackupRepository::NotFound)
    end

    def scoped_records
      Backup.for_destination(destination_key, **destination_coords)
    end

    def recorded?(filename)
      scoped_records.exists?(filename: filename)
    end

    def in_progress_cache_key
      "helios_#{destination_key}_backup_in_progress"
    end

    def pending_marker_content
      parse_pending_marker.first
    end

    # Parses the two-line pending marker into [filename, attempts]: the first
    # line is the planned filename, the optional second line the recording
    # retry counter (see bump_pending_attempts!). Returns [nil, 0] when the
    # marker is missing.
    def parse_pending_marker
      filename, attempts = ::File.readlines(pending_marker_path, chomp: true)
      [filename.presence, attempts.to_i]
    rescue Errno::ENOENT
      [nil, 0]
    end

    # `expected_filename` is BackupRunner.start's planned tar name, blank
    # from RestoreRunner.start — disambiguates backup vs restore on a clean
    # exit (no error file).
    def process_completion!(expected_filename) # rubocop:disable Metrics/MethodLength
      restore_err = read_error_file(RestoreRunner::ERROR_FILENAME)
      backup_err = read_error_file(BackupRepository::ERROR_FILENAME)
      logger.info(
        "[process_completion!] expected=#{expected_filename.inspect} " \
        "restore_err=#{restore_err.inspect} backup_err=#{backup_err.inspect}",
      )
      restore_failed = capture_error!(:restore, RestoreRunner::ERROR_FILENAME)
      backup_failed = capture_error!(:backup, BackupRepository::ERROR_FILENAME)

      kind =
        if backup_failed
          :backup
        elsif restore_failed || expected_filename.blank?
          :restore
        elsif record_backup!(expected_filename)
          # S3 uploader prunes earlier (so :pruning phase is visible); this
          # second call is a no-op in that path and handles every other adapter.
          prune!
          :backup
        elsif bump_pending_attempts!(expected_filename) < RECORDING_MAX_ATTEMPTS
          # Clean exit but the freshly written tar isn't readable yet — a
          # transient NAS/docker hiccup. Keep the marker so the next scheduler
          # tick retries, instead of losing the backup for good.
          return
        else
          # Gave up: the tar never appeared (OOM, host crash, no error file).
          RunnerLog.record_error!(:backup, I18n.t('backups.runner.errors.incomplete'))
          :backup
        end

      RunnerLog.record_finished!(kind)
      clear_pending!
    end

    def capture_error!(kind, filename) # rubocop:disable Naming/PredicateMethod
      message = read_error_file(filename)
      return false if message.blank?

      RunnerLog.record_error!(kind, message)
      remove_error_file!(filename)
      true
    end

    def upsert_backup_record(filename, bytes, archive)
      images = BackupRepository.images_from_config(archive.config)
      record = Backup.find_or_initialize_by(destination: destination_key, filename: filename,
                                            **destination_coords)
      record.assign_attributes(
        bytes: bytes,
        created_at: BackupRepository.created_at_from(filename) || Time.current,
        influxdb_image: images[:influxdb],
        postgresql_image: images[:postgresql],
        files: BackupRepository.summarize_entries(archive.entries),
      )
      record.save!
      record
    end

    # Bumps and returns the recording retry counter, persisted as the marker's
    # second line so it survives a HELIOS restart. Lets a transient failure to
    # read a finished tar be retried on later ticks instead of dropped.
    def bump_pending_attempts!(filename)
      attempts = parse_pending_marker.last + 1
      ::File.write(pending_marker_path, "#{filename}\n#{attempts}")
      attempts
    end
  end
end