2008-10-18

real street angels から mechanize を使って動画を取ってきてEmacsで見てみるよ

せっかく書いたから匿名でのせてみるよ

使い方は

  • 動画を取ってきたいよ
    • config.yamlユーザとかを設定するよ
    • ids.txt に取ってきたいIDを書くよ
    • sangels.bat を実行するよ
  • Emacs動画を見たいよ
    • sangels.el を load するよ
    • M-x sangels だよ

必要なものを gem で取ってくるにはこうすればいいよ

  • gem install -r log4r
  • gem install -r -v 0.6 hpricot
  • gem install -r mechanize

長すぎてelispが消えたから続きがあるよ

sangels.bat - 起動用バッチファイル

@echo off
setlocal
set WD=%~dp0
cd /d %WD%

ruby get_movies.rb
ruby get_images.rb
ruby create_m3u.rb

ruby

config.yaml - 設定ファイル
user: ユーザID
password: パスワード
ids_file: ids.txt
done_file: ids_done.txt
movies_dir: movies

log4r_config:
  pre_config:
    global: INFO
  loggers:
    - name: app
      type: Log4r::Logger
      level: INFO
      outputters:
        - STDOUT
        - FILE
  outputters:
    - name: STDOUT
      type: Log4r::StdoutOutputter
      formatter:
        type: Log4r::PatternFormatter
        pattern: "%d [%l] %C - %M"
        date_pattern: "%H:%M:%S"
    - name: FILE
      type: Log4r::FileOutputter 
      filename: "#{LOGDIR}/sangels.log"
      formatter:
        type: Log4r::PatternFormatter
        pattern: "%d [%l] %C - %M"
        date_pattern: "%Y-%m-%d %H:%M:%S"
get_movies.rb
require 'fileutils'
require 'logger'
require 'mechanize'

BASEDIR = File.dirname($0)
require "#{BASEDIR}/util"
require "#{BASEDIR}/sangels"

$config = load_config(BASEDIR)
prepare_logger(BASEDIR)
$log = new_logger("get_movies")
WWW::Mechanize.log = new_logger("mechanize")
WGet.log = $log

class IDFile
  def initialize(file)
    @file = file
    unless File.exist?(@file)
      Fileutils.touch(@file)
    end
  end

  def ids(contains_comment = nil)
    File.open(@file) {|io|
      io.to_a.map {|x|
        x.chomp
      }.select {|x|
        if x.empty?
          nil
        elsif contains_comment
          true
        else
          not /^\s*\#/ =~ x
        end
      }
    }
  end

  def add(id)
    ids = ids(true)
    unless ids.any? {|x| x == id}
      write(ids + [id])
    end
  end

  def delete(id)
    ids = ids(true)
    if ids.any? {|x| x == id}
      write(ids - [id])
    end
  end

  def write(ids)
    File.open(@file, "w") {|io|
      ids.each {|x| io.puts x}
    }
  end
end


$log.info("BEGIN #{$0} ================")
exit_code = 0
begin
  ids_file = IDFile.new($config.ids_file)
  done_file = IDFile.new($config.done_file)
  movies_dir = $config.movies_dir
  wget = WGet.new

  sangels = SAngels.new
  sangels.login($config.user, $config.password)
  ids_file.ids.each {|id|
    begin
      movies = sangels.movies(id)
    rescue SAngels::Movies::InvalidMoviesError
      $log.warn("invalid movie id: #{id}")
      next
    end
    dir = File.expand_path(id, movies_dir)
    movies.each {|link|
      wget.retrieve(link.href, dir)
    }
    expected = movies.movie_links.map{|x| File.basename(x.href)}
    actual = Dir.glob("#{dir}/*").map {|x| File.basename(x)}
    if (expected - actual).empty?
      done_file.add(id)
      ids_file.delete(id)
    end
  }
rescue => e
  $log.error(e)
  exit_code = 1
end

$log.info("END #{$0} (#{exit_code}) ================")
exit exit_code
get_images.rb
require 'fileutils'
require 'logger'
require 'mechanize'
require 'ostruct'

BASEDIR = File.dirname($0)
require "#{BASEDIR}/util"
require "#{BASEDIR}/sangels"

$config = load_config(BASEDIR)
prepare_logger(BASEDIR)
$log = new_logger("get_images")
WWW::Mechanize.log = new_logger("mechanize")
WGet.log = $log

$log.info("BEGIN #{$0} ================")
exit_code = 0
begin
  movies_dir = $config.movies_dir
  sangels = SAngels.new
  sangels.login($config.user, $config.password)
  thumbnails = sangels.thumbnails

  Dir.glob("#{movies_dir}/*").each {|dir|
    next unless File.directory? dir
    id = File.basename(dir)

    url = thumbnails.url(id)
    unless url
      $log.warn("#{id} is not found")
      next
    end
    path = File.expand_path("00_thumbnail#{File.extname(url)}", dir)
    next if File.exist? path

    $log.info("retrieving #{url}")
    thumbnail = thumbnails.get_file(id)
    File.open(path, "wb") {|io| io.write(thumbnail)}
  }
rescue => e
  $log.error(e)
  exit_code = 1
end

$log.info("END #{$0} (#{exit_code}) ================")
exit exit_code
create_m3u.rb
BASEDIR = File.dirname($0)
require "#{BASEDIR}/util"

$config = load_config(BASEDIR)
movies_dir = $config.movies_dir
Dir.glob("#{movies_dir}/*") {|dir|
  next unless File.directory? dir
  name = File.basename(dir)
  files = Dir.glob("#{dir}/*.wmv").sort

  File.open("#{movies_dir}/#{name}.m3u", "w") {|io|
    files.each {|file|
      io.puts "#{name}/#{File.basename(file)}"
    }
  }

  File.open("#{dir}/00_movies.m3u", "w") {|io|
    files.each {|file|
      io.puts "#{File.basename(file)}"
    }
  }
}
sangels.rb
require 'mechanize'
require 'hpricot'

BASEDIR = File.dirname($0)
require "#{BASEDIR}/util"

class SAngels
  HOST = "real2.s-angels.com"
  LOGIN_URL = "http://#{HOST}/member/"
  INFO_URL = "http://#{HOST}/teigaku/item.php"
  THUMBNAILS_URL = "http://#{HOST}/teigaku/"
  THUMBNAIL_URL = "http://#{HOST}/images/default/thumb/"

  def initialize()
    @agent = WWW::Mechanize.new
  end

  def login(user, password)
    login_form = @agent.get(LOGIN_URL).forms.find {|form|
      form.fields.any? {|field| field.name == "frmLoginid"}
    }
    login_form.frmLoginid = user
    login_form.frmPw = password
    @agent.submit(login_form)
  end

  def movies(id, no_validate = nil)
    Movies.new(@agent, id, !no_validate)
  end

  def thumbnails
    Thumbnails.new(@agent)
  end

  class Thumbnails
    def initialize(agent)
      @agent = agent
      doc = Hpricot(@agent.get_file(THUMBNAILS_URL))
      elems = doc.search("div[@class=realthum]/a")
      @links = Hash(
        elems.map {|elem|
          href = elem["href"]
          id = $1 if /ID=(.+)/ =~ href
          url = elem.search("img")[0]["src"]
          [id, url]
        })
    end

    def get_file(id)
      @agent.get_file(url(id))
    end

    def url(id)
      @links[id]
    end

    def exist?(id)
      url(id)
    end
  end

  class Movies
    class InvalidMoviesError < StandardError
    end

    def initialize(agent, id, no_validate)
      @agent = agent
      @id = id
      if !no_validate &amp;&amp; !valid?
        raise InvalidMoviesError
      end
    end

    def info_page_url
      "#{INFO_URL}?ID=#{@id}"
    end

    def info_page
      @agent.get(info_page_url)
    end

    def movies_page
      @agent.click(info_page.links.find {|link| /P=10/ =~ link.href})
    end

    def movie_links
      movies_page.links.select {|link|
        /wmv$/ =~ link.href
      }.sort {|a, b|
        File.basename(a.href) <=> File.basename(b.href)
      }
    end

    def valid?
      info_page.uri.to_s == info_page_url
    end

    def each(&amp;block)
      orig_links = movie_links
      orig_links.each {|orig_link|
        link = movie_links.find {|l| File.basename(l.href) == File.basename(orig_link.href)}
        block.call(link)
      }
    end
  end
end
util.rb
require 'log4r'
require 'log4r/yamlconfigurator'
require 'singleton'
require 'fileutils'
require 'ostruct'

def Hash(a)
  Hash[*a.flatten]
end

def load_config(basedir)
  OpenStruct.new(File.open("#{basedir}/config.yaml") {|io| YAML.load(io)})
end

def new_logger(name)
  Log4r::Logger.new("app::#{name}")
end

def prepare_logger(basedir, logdir = nil)
  logdir ||= basedir
  Log4r::YamlConfigurator["LOGDIR"] = logdir
  Log4r::YamlConfigurator.load_yaml_file("#{basedir}/config.yaml")
end

class NullObject
  include Singleton
  def method_missing(message, *arg)
    NullObject.singleton
  end
end

class WGet
  class << self
    attr_accessor :log
    def initialize
      super
      @log = NullObject.singleton
    end
  end
  
  def log
    self.class.log
  end

  def retrieve(url, dir)
    FileUtils.mkdir_p(dir)
    file = File.expand_path(File.basename(url), dir)
    if File.exist?(file)
      log.info("already retrieved #{url}")
      return true
    end

    tmp = "#{file}.part"
    log.info("retrieving #{url}")
    ret = system("wget", "-c", "-O", tmp, url)
    if ret
      log.info("retrieving succeeded #{url}")
      File.rename(tmp, file)
    else
      if $? == 0x020000 # Ctrl-C
        exit($?)
      else
        log.error("retrieving failure #{url} (#{$?})")
      end
    end
    return ret
  end
end
  • http://anond.hatelabo.jp/20081018072817 の続きだよ

    これでおしまいだよ elisp sangels.el (require 'cl) ; for cl-seq(defvar sangels-movies-dir nil)(defvar sangels-player &quot;c:/Program Files/GRETECH/GomPlayer/GOM.exe&quot;)(defvar sangels-sort-by 'sangels-sort-by-rate)(defvar sangels-rate...

    • http://anond.hatelabo.jp/20081018073905

      増田なんて500PVもない位なのに、そんな特定層にしかわからないような ネタ投げても誰も使わないぞ

    • http://anond.hatelabo.jp/20081018073905

      俺もここまでしっかりしてないが サイト特化型のダウンローダーをスクリプト言語で書いてるけど インタフェース部分にemacs使うのは凄いなぁって思った こういうインタフェースをサラ...

  • http://anond.hatelabo.jp/20081018072817

    ずっと昔から作っててそのままえいやーと公開した感じがあってよい

記事への反応(ブックマークコメント)

アーカイブ ヘルプ
ログイン ユーザー登録
ようこそ ゲスト さん