Update from Subversion Repository Recipe

This is a script/daemon that updates a set of puppetmaster directories(manifests,files,etc) from a subversion repository, and will push those updates to additional puppetmasters via rsync+ssh.

It expects the subversion repository to setup in a trunk/tag format(though that's not necessary here, it's just the way that I use it). For instance:

/puppet
/puppet/trunk
/puppet/trunk/files
/puppet/trunk/manifests
....
/puppet/tags
/puppet/tags/prod-YYYYMMDDHHmmss

For the script below to work, /puppet-dev and /puppet-prod must have an initial check done ahead of time.

It'll do a "lite" query every 5 seconds, and perform updates if it detects something.

This script is most useful if you can't trigger an update via a commit hook.

puprepod

#!/usr/bin/env ruby

CHECKFREQ  = 5
USER       = "puppet"
PASSWORD   = "secret"

PUPPETDIRS = {
  "/puppet-dev" => nil,
  "/puppet-prod" => "http://svnserver/repo/puppet/tags",
}
PUPPETMASTERS = [
  "puppetmaster2",
  "puppetmaster3",
  "puppetmaster4",
]

require 'syslog'
require 'logger'

module UpdateLogger
  @@log = nil
  debug = false
  if debug
    @@log = Logger.new(STDOUT)
    @@log.level = Logger::INFO
  else
    @@log = Syslog.open("puprepod", Syslog::LOG_PID | Syslog::LOG_NDELAY)
  end

  def UpdateLogger::log
    @@log
  end

end

class RunCmd
  attr_reader :status, :output, :cmd

  def initialize(cmd)
    @cmd    = cmd
    @status = nil
  end

  def run
    @output = `#{@cmd} 2>&1`
    @status = $?
  end
end

class RepoError < StandardError; end
class RepoCheckError < RepoError; end
class RepoUpdateError < RepoError; end
class RepoChecker
  attr_reader :workingcopy

  def initialize(workingcopy)
    @workingcopy    = workingcopy
  end

  def get_repo_version
    UpdateLogger.log.debug("#{@workingcopy}: Getting repo version")
    cmd = RunCmd.new("/usr/bin/svn stat -Nu --username #{USER} --password #{PASSWORD} #{@workingcopy}")
    cmd.run
    raise RepoCheckError.new(cmd.output) if cmd.status.exitstatus != 0
    m = /^Status against revision:\s+(\d+)\n/.match(cmd.output)
    raise RepoCheckError.new("Unable to determine repository version: #{cmd.output}") if m.nil?
    UpdateLogger.log.debug("#{@workingcopy}: repo version: #{m[1]}")
    m[1].to_i
  end

  def get_working_copy_version
    UpdateLogger.log.debug("#{@workingcopy}: Getting workingcopy version")
    cmd = RunCmd.new("/usr/bin/svn info #{@workingcopy}")
    cmd.run
    raise RepoCheckError.new(cmd.output) if cmd.status.exitstatus != 0
    m = /^Revision:\s+(\d+)$/.match(cmd.output)
    raise RepoCheckError.new("Unable to determine workingcopy version: #{cmd.output}") if m.nil?
    UpdateLogger.log.debug("#{@workingcopy}: workingcopy version: #{m[1]}")
    m[1].to_i
  end

  def uptodate?
    get_working_copy_version == get_repo_version
  end

  def update
    UpdateLogger.log.debug("#{@workingcopy}: Updating")
    cmd = RunCmd.new("/usr/bin/svn update --username #{USER} --password #{PASSWORD} #{@workingcopy}")
    cmd.run
    raise RepoUpdateError.new(cmd.output) if cmd.status.exitstatus != 0
    UpdateLogger.log.debug("/usr/bin/svn update --username #{USER} --password ******** #{@workingcopy} output:\n  #{cmd.output.gsub("\n", "\n  ")}")
    UpdateLogger.log.info("#{@workingcopy}: updated to #{get_working_copy_version}")
    cmd.output
  end

end

class TagError < StandardError; end
class TagCheckError < TagError; end
class TagUpdateError < TagError; end
class TagFollower
  attr_reader :workingcopy, :tagrepodir

  def initialize(workingcopy, tagrepodir)
    @workingcopy = workingcopy
    @tagrepodir  = tagrepodir
  end

  def get_repo_tag
    UpdateLogger.log.debug("#{@workingcopy}: Getting repo tag")
    cmd = RunCmd.new("/usr/bin/svn ls -v --username #{USER} --password #{PASSWORD} #{@tagrepodir} | latesttag")
    cmd.run
    raise TagCheckError.new(cmd.output) if cmd.status.exitstatus != 0
    cmd.output.strip
  end

  def get_working_copy_tag
    UpdateLogger.log.debug("#{@workingcopy}: Getting workingcopy tag")
    cmd = RunCmd.new("/usr/bin/svn info #{@workingcopy}")
    cmd.run
    raise TagCheckError.new(cmd.output) if cmd.status.exitstatus != 0
    m = /^URL: (\S+)$/.match(cmd.output)
    raise TagCheckError.new("Unable to determine tag directory: #{cmd.output}") if m.nil?
    raise TagCheckError.new("Tag repo does not match @tagrepodir: #{cmd.output}") if m[1][0,@tagrepodir.size] != @tagrepodir
    tag = m[1][@tagrepodir.size+1 .. -1] + "/"
    UpdateLogger.log.debug("#{@workingcopy}: workingcopy tag: #{tag}")
    tag
  end

  def uptodate?
    get_working_copy_tag == get_repo_tag
  end

  def update
    UpdateLogger.log.debug("#{@workingcopy}: Updating")
    cmd = RunCmd.new("/usr/bin/svn switch --username #{USER} --password #{PASSWORD} #{@tagrepodir}/#{get_repo_tag} #{@workingcopy}")
    cmd.run
    raise TagUpdateError.new(cmd.output) if cmd.status.exitstatus != 0
    UpdateLogger.log.debug("/usr/bin/svn switch --username #{USER} --password ******** #{@tagrepodir}/#{get_repo_tag} #{@workingcopy}")
    UpdateLogger.log.info("#{@workingcopy}: updated to #{get_working_copy_tag}")
    cmd.output
  end

end

class RsyncError < StandardError; end
class Rsyncer

  def initialize(source, target)
    @source = source
    @target = target
  end

  def run
    UpdateLogger.log.debug("#{@source}: #{@target} syncing")
    cmd = RunCmd.new("rsync -v -aHz --delete --exclude .svn #{@source}/ root@#{@target}:#{@source}/")
    cmd.run
    raise RsyncError.new(cmd.output) if cmd.status.exitstatus != 0
    UpdateLogger.log.debug("rsync -v -aHz --delete --exclude .svn #{@source}/ root@#{@target}:#{@source}/ output:\n  #{cmd.output.gsub("\n", "\n  ")}")
    UpdateLogger.log.info("#{@source}: #{@target} synced")
  end

end

class WorkHorse

  def initialize
    @repos      = PUPPETDIRS
    @repochecks = {}
    @rsyncs     = {}

    @repos.keys.each do |d|
      if @repos[d].nil?
        @repochecks[d] = RepoChecker.new(d)
      else
        @repochecks[d] = TagFollower.new(d,@repos[d])
      end
    end
    @repos.keys.each do |d|
      @rsyncs[d] ||= []
      PUPPETMASTERS.each do |m|
        @rsyncs[d] << Rsyncer.new(d,m)
      end
    end

    UpdateLogger.log.info("Refreshing everything")
    @repos.keys.each do |d|
      begin
        @repochecks[d].update
        @rsyncs[d].each do |rs|
          begin
            rs.run
          rescue RsyncError => e
            UpdateLogger.log.err("Unable to rsync:\n  %s"%(e.to_s.gsub("\n", "\n  ")))
          end
        end
      rescue RepoUpdateError => e
        UpdateLogger.log.err("Unable to run update:\n  %s"%(e.to_s.gsub("\n", "\n  ")))
      rescue TagUpdateError => e
        UpdateLogger.log.error("Unable to run update:\n  %s"%(e.to_s.gsub("\n", "\n  ")))
      end
    end
  end

  def main
    loop do
      repos_to_rsync = []
      UpdateLogger.log.debug("Main Loop")

      @repos.keys.each do |d|
        rc = @repochecks[d]
        begin
          if not rc.uptodate?
            UpdateLogger.log.debug("#{d}: Not up to date.  Updating.")
            res = rc.update
            if res[0,12] != "At revision "
              UpdateLogger.log.debug("#{d}: Has updates.  Rsync scheduled.")
              repos_to_rsync << d
            else
              UpdateLogger.log.info("#{d}: No new updates.")
            end
          end
        rescue RepoCheckError => e
          UpdateLogger.log.err("Unable to run check:\n  %s"%(e.to_s.gsub("\n","\n  ")))
        rescue RepoUpdateError => e
          UpdateLogger.log.err("Unable to run update:\n  %s"%(e.to_s.gsub("\n", "\n  ")))
        rescue TagCheckError => e
          UpdateLogger.log.err("Unable to run check:\n  %s"%(e.to_s.gsub("\n","\n  ")))
        rescue TagUpdateError => e
          UpdateLogger.log.err("Unable to run update:\n  %s"%(e.to_s.gsub("\n", "\n  ")))
        end
      end

      repos_to_rsync.each do |d|
        UpdateLogger.log.debug("#{d}: Rsyncs starting")
        @rsyncs[d].each do |rs|
          begin
            rs.run
          rescue RsyncError => e
            UpdateLogger.log.err("Unable to rsync:\n  %s"%(e.to_s.gsub("\n", "\n  ")))
          end
        end
        UpdateLogger.log.debug("#{d}: Rsyncs done")
      end

      UpdateLogger.log.debug("Sleeping for #{CHECKFREQ} seconds...")
      sleep(CHECKFREQ)
    end
  end

  def spawn
    pid = fork do
      Signal.trap('HUP', 'IGNORE')
      main
    end
    Process.detach(pid)
  end

end

a = WorkHorse.new()
a.spawn()

latesttag

#!/usr/bin/env ruby

def get_latest_tag(svn_output)
  tag = nil
  svn_output.each_line do |l|
    newtag = [l.split()[0].to_i, l.split()[5]]
    next if newtag[1] !~ /^prod-(\d{14})\/$/
    if tag.nil?
      tag = newtag
    else
      tag = newtag if newtag[0] > tag[0]
    end
  end
  return tag[1]
end

puts get_latest_tag(STDIN)
exit(0)