Recovering Rails Data Using Production.log

So you didn’t set up automatic backups and need to recover your data? Don’t try digging it out of mysql, look no further than your rails production log. OK, try a bit with mysql, because the rails production log is a terrible way to recover your data. Only use as a last resort, but when data don’t exist anywhere else, this is what you have to do. While simple in concept (scrape the log, updated the database using ActiveRecord) this was very trick in implementation. I know my code could use some refactoring, but it is working pretty well right now.

So here is my code if anyone wants to do something similar.

I had a production log in the general format of:


Processing AttendanceController#update_exercises (for 76.105.103.226 at 2010-07-07 20:33:26) [POST]
  Session ID: 06af5da80fc41be243e42254cb776212
  Parameters: {"commit"=>"Record User Scores", "meeting_id"=>"3388", "exertions"=>{"exercise_note_437"=>"7:00 AM", "exercise_score_884"=>"6:52", "exercise_score_1120"=>"9:29", "exercise_score_1175"=>"13:21", "exercise_score_268"=>"8:07", "exercise_score_433"=>"6:29", "exercise_note_1200"=>"", "exercise_note_1190"=>"", "exercise_score_741"=>"11:02", "exercise_note_1191"=>"", "exercise_score_544"=>"8:37", "exercise_score_918"=>"***", "exercise_note_1124"=>"", "exercise_note_1069"=>"", "exercise_score_479"=>"9:18", "exercise_score_908"=>"8:04", "exercise_score_1012"=>"7:34", "exercise_score_105"=>"8:43", "exercise_note_417"=>"", "exercise_id"=>"25", "exercise_note_968"=>"", "exercise_note_913"=>"", "exercise_score_1200"=>"8:50", "exercise_note_1115"=>"", "exercise_score_437"=>"7:16", "exercise_score_1124"=>"9:32", "exercise_note_705"=>"", "exercise_score_1190"=>"12:26", "exercise_note_342"=>"", "exercise_score_1069"=>"9:34", "exercise_score_1191"=>"10:45", "exercise_note_1095"=>"", "exercise_note_872"=>"", "exercise_score_1115"=>"8:12", "exercise_score_417"=>"9:00", "exercise_note_884"=>"", "exercise_score_968"=>"9:02", "exercise_score_913"=>"7:59", "exercise_note_268"=>"", "exercise_note_433"=>"", "exercise_note_1130"=>"", "exercise_note_1031"=>"", "exercise_score_342"=>"9:26", "exercise_note_741"=>"", "exercise_note_1120"=>"", "exercise_note_544"=>"", "exercise_note_1175"=>"", "exercise_note_918"=>"", "exercise_score_1095"=>"6:59", "exercise_score_705"=>"7:29", "exercise_note_479"=>"", "exercise_note_908"=>"", "exercise_score_872"=>"8:25", "exercise_note_105"=>"", "exercise_score_1130"=>"8:14", "exercise_score_1031"=>"7:24", "exercise_note_1012"=>""}}
Redirected to actionindexid3388
Completed in 536ms (DB: 999) | 302 Found [http://www.fitwit.com/attendance/update_exercises]


Processing AttendanceController#index (for 76.105.103.226 at 2010-07-07 20:33:27) [GET]
  Session ID: 06af5da80fc41be243e42254cb776212
  Parameters: {"id"=>"3388"}
Rendering template within layouts/application
Rendering attendance/index
Completed in 195ms (View: 172, DB: 6) | 200 OK [http://www.fitwit.com/attendance/index/3388]

So I wrote this code to fix it. Basically, well, it’s complicated, if you have any questions, email me.


#!/usr/bin/env /Users/Tim/Sites/fitwit/script/runner

class CommandString
  attr_accessor :date, :method, :command
  
  def initialize(date, method, command)
    @date = date
    @method = method
    @command = command
  end
end

class ExerciseArray
  attr_accessor :exercise_id, :time_slot_id, :meeting_id, :values_hash

  def initialize(exercise_id, time_slot_id, meeting_id, values_hash)
    @exercise_id = exercise_id
    @time_slot_id = time_slot_id
    @meeting_id = meeting_id
    @values_hash = values_hash
  end
end

class RecoverData "Update", ("user_ids"=>[.*], "meeting_id"=>".*")/
        params = gen_hash("{#{$1}}")
        j+=1
        c = CommandString.new(@the_date,'take_attendance', params)
        commands <"(d+)", "commit"=>"Apply to ([ws]*)", "meeting_times"=>[(.*)], "id"=>"(d+)"}/
        boot_camp_id = $1.to_i
        if boot_camp_id = 40
          commit = "Apply to #{$2}"
          meeting_times = eval("[#{$3}]")
          time_slot_id = $4.to_i
          k+=1
          c = CommandString.new(@the_date,'manage_meetings', [time_slot_id, commit, meeting_times, boot_camp_id]) if boot_camp_id == 40
          commands <"Record User Scores", "meeting_id"=>"([0-9]+)", "exertions"=>({.*})}/
        meeting_id = $1.to_i
        #puts meeting_id
        attendees = gen_hash($2)
        i+=1
        c = CommandString.new(@the_date,'update_exercises', [meeting_id, attendees])
        commands < mt)
            ts.meetings < mt)
          @time_slot.meetings < user_id, :meeting_id => meeting_id) if submitted_user_new?(user_id, existing_attendee_ids)
          end
          # now deletes (for each existing user, see if they are excluded)
          existing_attendee_ids.each do |existing_id|
            if should_delete?(existing_id, post_user_ids)
              @meeting_user = MeetingUser.find_by_user_id_and_meeting_id(existing_id, meeting_id)
              @meeting_user.destroy
            end
          end
          puts "successful! attendance taken"
        else
          puts "bootcamp 41 or something, so no attendance taken"
        end
      else
        puts "attendance: #{meeting_id}"
      end
    else
      # need to raise exception
      puts "meeting id is nil for #{params.inspect}"
    end
  end

  def update_exercises(meeting_id, attendees)
    puts "updating exercises"
    # this is where we update our exercises which is the exertions table
    # the meeting _might_ not exist
    #begin
      if Meeting.exists?(meeting_id)
        meeting = Meeting.find(meeting_id)
        if meeting.time_slot.boot_camp.id == 40
          puts "!! found #{meeting_id}"
          users =  attendees.keys.map{|m| $1 if m.match(/exercise_score_(d+)$/) }.delete_if{|n| n.nil?} #  meeting.meeting_users
          exercise = Exercise.find(attendees["exercise_id"])
          users.each do |u_id|
            u = User.find(u_id)
            unless attendees["exercise_score_#{u.id}"].blank?
              ex = Exertion.new
              mus = MeetingUser.all(:conditions => ["meeting_id = ? and user_id = ?", meeting.id, u.id])
              if mus.empty? # then, we need to create an attendence
                mu = MeetingUser.create(:user_id => u.id, :meeting_id => meeting.id)
                puts "created attendance event for #{u.full_name}" 
              else
                mu = mus.first
              end
              ex.meeting_user_id = mu.id
              ex.exercise_id = exercise.id
              ex.score = attendees["exercise_score_#{mu.user.id}"]
              raise "user: #{mu.user.id} mu: #{mu.id} ex: #{exercise.id}" if attendees["exercise_score_#{mu.user.id}"] == ''
              ex.notes = attendees["exercise_note_#{mu.user.id}"]
              ex.save
            end
          end
          puts "Recorded scores for #{exercise.name}"
          puts meeting.meeting_date
        else
          puts "wrong boot-camp #{meeting.time_slot.boot_camp.id}"
        end
      else
        puts "exercises:  #{meeting_id}"
      end
    #rescue
     
     # raise "bad: can't pull it off"
    #end
  end

  def submitted_user_new?(submitted_id,existing_ids)
    # if submitted_id is not in existing id's it is new
    !existing_ids.include?(submitted_id)
  end

  def should_delete?(existing_id,submitted_ids)
    #if existing id is not in submitted id it should be deleted
    !submitted_ids.include?(existing_id)
  end

end

BootCamp.find(40).time_slots.map{|ts| ts.meetings.destroy_all}

r = RecoverData.new()
commands = r.get_data
puts "about to run commands"
attendance_cursor = {98 => 0, 99 => 0, 100 => 0, 101 => 0}
exercise_cursor = {98 => 0, 99 => 0, 100 => 0, 101 => 0}
ea = []
commands.each do |c|
  case c.method
  when "take_attendance"
    user_id = c.command["user_ids"].first
    ts_all = User.find(user_id).registrations.map{|reg| reg.time_slot}
    ts = ts_all.select{|ts| ts if ts.boot_camp_id == 40}.first
    unless ts.nil?
      ts_id = ts.id
      bc = ts.boot_camp.id
      meeting_id = ts.meetings.map{|m| m.id}[[attendance_cursor[ts_id],23].min]
      #puts "meeting id is #{meeting_id}"
      attendance_cursor[ts_id]+=1
      puts "#{ts_id}, #{bc}, #{c.command["meeting_id"]}, #{c.date}, #{c.method}, #{c.command["user_ids"].to_sentence}"
      r.do_attendance(c.command, meeting_id)
    else
      puts "no timeslots for #{user_id}, but they are in bootcamps #{ts_all.map{|ts| ts.boot_camp.id}.to_sentence}"
    end
  when "manage_meetings"
    #  [time_slot_id, commit, meeting_times, boot_camp_id]
    if c.command.last == 40
      #puts "#{c.date}, #{c.method}, #{c.command.to_sentence}"
      r.manage_meetings(c.command[0], c.command[1], c.command[2], c.command[3])
    end
  when "update_exercises"
    # [meeting_id, attendees]
    params = c.command[1]
    params.first.first.match(/(d+)$/)
    user_id = $1 || 399 # this is to account for a wierd record that has an exercise if 
    u = User.find(user_id)
    ts = u.registrations.map{|reg| reg.time_slot}.select{|ts| ts.boot_camp.id == 40}.first
    ex_id = params["exercise_id"]
    puts "!! #{ex_id}"
    unless ts.nil?
      ts_id = ts.id
      puts "time slot: #{ts_id}"
      m_index = [exercise_cursor[ts_id],23].min
#      puts "m index #{m_index}"
#      puts "meetings #{ts.meetings.inspect}"
      meeting_id = ts.meetings.map{|m| m.id}[m_index]
#      puts "meeting id: #{meeting_id}"
      exercise_cursor[ts_id]+=1
      # :exercise_id, :time_slot_id, :meeting_id, :values_hash
      ea << ExerciseArray.new(ex_id, ts_id, meeting_id, params)
      #puts "#{c.date}, #{c.method}, #{meeting_id}, #{c.command[1].inspect}"
      #r.update_exercises(meeting_id, c.command[1])
    else
      puts "all time slots are empty for bc 40 for user #{u.id} (they are in another boot_camp)"
    end
  end
end
puts "doing it"
ea.group_by{|e| [e.exercise_id, e.time_slot_id] }.each do |e|
  values_hash = {}
  meeting_id = e[1].first.meeting_id
  puts e.inspect
  e[1].each do |poopa|
    values_hash = values_hash.merge(poopa.values_hash)
  end
  puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
  puts "#{meeting_id}, #{values_hash.inspect}"
  puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
  r.update_exercises(meeting_id,values_hash)
end

Leave a Reply