#!/usr/bin/env ruby

RAILS_ENV = 'production' if ARGV.delete('-p') or ARGV.delete('--production')

require File.dirname(__FILE__)+"/../config/environment"
require "kconv"
require 'opendapdir'
require 'localdir'
require "optparse"

# === 参照先のノードが未保存の場合、後で参照先のノードが保存された場合に
# 参照を適切にDBに設定するためのメソッド
# register_image, register_knowledgeの最後にこれを呼ぶ
ToDo = Hash.new   #  path => proc
def after_save(obj)
  if proc = ToDo[obj.path]
    proc.call(obj.node)
  end
end

class OptionParser
  # Changed to raise error. (Originally, invalid options
  # are just warned, which is easily overlooked.)
  def warn(mesg = $!)  
    raise mesg
  end
end


#################################################
module NumRu

  # == To be merged into the original GPhys distribution
  # Geospatio-temporal coordinate handing
  class GPhys

    def lon_lat_t_ranges
      geocoords = lon_lat_t_coords
      georanges = Hash.new
      geocoords.each do |nm,crd|
        georanges[nm] = [crd.max, crd.min]
      end
      georanges
    end

    def lon_lat_t_coords
      geocoords = Hash.new   # will ba Hash of :lon, :lat, and :time

      #< from the coordinate variables >
      for dim in 0...self.rank
        crd = add_lon_lat_t_coords(self.coord(dim), geocoords)
      end

      #< from the "coordinates" attributes (CF convenction) >
      if geocoords.length!=3 and data.respond_to?(:file) and \
               (nms = self.get_att("coordinates"))
        nms.split.each do |nm|
          if data.file.var(nm)
            crd = GPhys::IO.open(data.file, nm).data
            geocoords = add_lon_lat_t_coords(crd, geocoords)
          end
        end
      end

      #< from scalar variables >
      if geocoords.length!=3 && data.respond_to?(:file)
        data.file.var_names.each do |nm|
          if /(lon|lat|time)/i =~ nm
            crd = GPhys::IO.open(data.file, nm).data
            if crd.length==1
              geocoords = add_lon_lat_t_coords(crd, geocoords) 
            end
          end
        end
      end

      geocoords.each do |nm,crd|
        if nm==:lon or nm==:lat
          un = crd.units
          if un and /(^degree|^\s*$)/i !~ un.to_s
            geocoords[nm] = un.convert2(crd,Units['degrees'])
          end
        end
      end

      geocoords
    end

    ########################
    private
    def add_lon_lat_t_coords(coord, geocoords)
      unt = coord.units.to_s
      if !geocoords[:lon] && /^degrees?_east$/i =~ unt
        geocoords[:lon] = coord
      elsif !geocoords[:lat] && /^degrees?_north$/i =~ unt
        geocoords[:lat] = coord
      elsif !geocoords[:time] &&  /since/ =~ unt
        geocoords[:time] = coord
      end
      nm = coord.name
      if !geocoords[:lon] && /^lon(gitude)?$/i =~ nm
        geocoords[:lon] = coord
      elsif !geocoords[:lat] && /^lat(itude)?$/i =~ nm
        geocoords[:lat] = coord
      end
      geocoords
    end

  end

  # == To be  merged into the original GPhys distribution 
  # for unumeric.rb
  class UNumeric
    def to_datetime
      if /(.*) *since *(.*)/ =~ units.to_s
        stun = $1
        since = DateTime.parse($2)
        tun = Units[stun]
        sec = tun.convert( val, Units['seconds'] ).round + 1e-1
               # ^ take "round" to avoid round error in sec
               # (Note that %S below takes floor of seconds).
        datetime = since + (sec/86400.0)
      else
        nil
      end
    end
  end
end
##################################################

# Hash with String / Regexp keys
class RegHash < Hash
  def []=(key,val)
    if key.is_a?(String) && /[\*%\?]/ =~ key  # wild cards
      key = Regexp.new( '^' + key.gsub(/[\*%]/,'.*').gsub(/\?/,'.') + '$' )
    end
    super(key,val)
  end
  def [](key)
    (v=super(key)) && (return(v))
    keys.each do |k|
      if key.is_a?(String) && k.is_a?(Regexp) && k =~ key
        return(super(k))
      else
        nil
      end
    end
    nil
  end
  def delete(key)
    (v=super(key)) && (return(v))
    keys.each do |k|
      if key.is_a?(String) && k.is_a?(Regexp) && k =~ key
        return(super(k))
      else
        nil
      end
    end
    nil
  end
end

module MSG
  module_function
  @@verbose_level = 1   # normal
  def to_quiet; @@verbose_level = 0; end
  def to_normal; @@verbose_level = 1; end
  def to_verbose; @@verbose_level = 2; end
  def verbose(msg, altmsg=nil)
    if @@verbose_level == 2
      print(msg) 
    elsif altmsg
      print(altmsg) 
    end
    STDOUT.flush
  end
  def normal(msg, altmsg=nil)
    if @@verbose_level >= 1
      print(msg) 
    elsif altmsg
      print(altmsg) 
    end
    STDOUT.flush
  end
end

##################################################
include NumRu

Var_Meta_To_Register = RegHash.new

def open_dir(path)
  case path
  when /^http.*(\/dap\/|\/nph-dods\/)/
    dir = OPeNDAPDir.open(path)
  when /^\//, /^[a-zA-Z]:/
    dir = LocalDir.open(path)
  else
    raise ArgumentError, "Unsupported kind of path: #{path}"
  end
  dir
end

GFDNAVI_IGNORED_DIRS_PAT = Regexp.union( *GFDNAVI_IGNORED_DIRS.collect{|igd|
                           igd.is_a?(String) ? Regexp.new('^'+igd+'$') : igd} )
IMAGE_PAT = /(\.png$|\.jpg$|\.jpeg$|\.gif$|\.tiff$)/i
KNOWLEDGE_PAT = /\.knlge$/i

def parse_dir(dir, parent=nil, mtime=nil)
  #plain_file_paths = dir.plain_file_paths   # for yaml, png files etc

  rdir,flag = register_dir(dir.path, parent, mtime)

  dir.each_gphys_file do |path, mtime, size, klass|
    Directory.transaction do
      rd,flag = register_dir(path, rdir, mtime, size, true)
      if flag
        begin
          file = klass.open(path)
          begin
            GPhys::IO.var_names_except_coordinates(file).each do |vname|
              register_var(file,vname,rd,mtime)
            end
          rescue
            if dir.is_a?( NumRu::OPeNDAPDir )
              # just to warn, because one cannot correct the problem in general
              warn "*** Error occured while processing remote file #{file.path} : #{$!.to_s}" 
            else
              raise $!
            end
          ensure
            file.close
          end
        rescue
          if !OPTS[:ignore_errors]
            raise $!
          end
        end
      end
    end
  end


  Var_Meta_To_Register.clear

  dir.each_file(IMAGE_PAT) do |path, mtime, size|
    if File.exist?( yml = path+'.yml' )
      meta = YAML.load( File.read(yml) )
    else
      meta = nil
    end
    register_image(path,meta,mtime,size)
  end
  
  # knowledge
  dir.each_file(KNOWLEDGE_PAT) do |path, mtime, size|
    unless /\.\d\.knlge$/  =~ path
      if File.exist?(path)
        meta = YAML.load( File.read(path) )
      else
        meta = nil
      end
      register_knowledge(path,meta,mtime,size)
    end
  end

  unless /nus$/ =~ dir.path
    dir.each_dir do |subdir, mtime|
      if GFDNAVI_IGNORED_DIRS_PAT !~ subdir.name
        parse_dir(subdir, rdir, mtime)
      end
    end
  end

  rdir
end

def register_var(file,vname,parent,mtime)
  ## print "   registering  #{vname}\n"

  if String === file
    file = Dir[File.join(GFDNAVI_DATA_PATH,file)].sort
    rpath_var = vname
    vname = File.basename(vname)
  else
    rpath = file.path.sub(GFDNAVI_DATA_PATH,"")
    rpath_var = File.join(rpath,vname)
    rpath_dir = rpath
    vname.split("/")[0..-2].each{|dname|
      rpath_dir = File.join(rpath_dir,dname)
      parent, = register_dir(rpath_dir, parent, mtime)
    }
  end

  unless var = Variable.find(:first, :conditions=>["path=?",rpath_var])
    var = Variable.new
  end

  if Array === file
    file.each{|f|
      var.actual_files.push ActualFile.new(:path=>f.sub(GFDNAVI_DATA_PATH,""))
    }
  else
    var.file = rpath
  end

  var.name = File.basename(vname)
  var.mtime = mtime
  var.path = rpath_var
  var.parent = parent.node
  gphys = NumRu::GPhys::IO.open(file,vname)
  var.size = gphys.length
  node = var.node
  gphys.att_names.each{|aname|
    register_kattr(aname, gphys.get_att(aname), node)
  }
  if (meta = Var_Meta_To_Register[rpath_var])  # substitution, not ==
    register_meta(meta, node)
  end
  if !node.remote? or OPTS[:stremote] # too slow for remote
    georanges = gphys.lon_lat_t_ranges
    if georanges.length > 0
      sta = SpatialAndTimeAttribute.new
      sta.node = node
      if r=georanges[:lon]
        sta.longitude_lb, sta.longitude_rt = r.collect{|un| un.val}
      end
      if r=georanges[:lat]
        sta.latitude_lb, sta.latitude_rt = r.collect{|un| un.val}
      end
      if r=georanges[:time]
        sta.starttime,sta.endtime = r.collect{|un| un.to_datetime}
      end
      node.spatial_and_time_attributes.push(sta)
    end
  end
  unless var.save
    warn "failed to register variable: #{rpath}"
    warn "  #{var.errors.full_messages}"
  end
  MSG.verbose("registrated variable: #{rpath}\n")
end

def register_image(path,meta,mtime,size)
  rpath = path.sub(GFDNAVI_DATA_PATH,"").sub(File.dirname(GFDNAVI_USER_PATH),"")
  rpath = '/' if rpath == ''   # local root directory
  node = Node.find(:first, :conditions=>["path=?",rpath])
  if node
    if OPTS[:force] or t_comp(mtime, node.mtime) or !node.image?
      node.destroy   # for test
      node = nil     # for test
    else
      MSG.normal(".",".")
      return nil
    end
  end
  unless node
    img = Image.new
    img.name = File.basename(rpath)
    img.path = rpath
    img.mtime = mtime
    img.size = size
    register_meta(meta, img.node) if meta
    unless img.save
      warn "failed to register an image file (#{rpath})"
      warn "  #{img.errors.full_messages}"
    end
    MSG.verbose("\nregister directory: #{rpath}")
  end
  after_save(img)
  nil
end

def register_knowledge(path, meta, mtime, size)
  rpath = path.sub(GFDNAVI_DATA_PATH,"").sub(File.dirname(GFDNAVI_USER_PATH),"")
  rpath = '/' if rpath == ''   # local root directory
  node = Node.find(:first, :conditions=>["path=?",rpath])
  if node
    if OPTS[:force] or t_comp(mtime, node.mtime) or !node.knowledge?
      node.destroy   # for test
      node = nil     # for test
    else
      MSG.normal(".",".")
      return nil
    end
  end
  unless node
    knowledge = Knowledge.new
    knowledge.name = File.basename(rpath)
    knowledge.path = rpath
    knowledge.mtime = mtime
    knowledge.size = size
    register_meta(meta, knowledge.node) if meta
    unless knowledge.save
     warn "failed to register a knowledge file (#{rpath})"
     warn "  #{knowledge.errors.full_messages}"
    end
    MSG.verbose("\nregister directory: #{rpath}")
  end
  after_save(knowledge)
  nil
end

def t_comp(a,b)
  return(nil) if a.nil? or b.nil?
  if a.class == b.class
    a > b
  end
end

def register_dir(path, parent, mtime, size=nil, plain_file=false, kas=nil)
  rpath = path.sub(GFDNAVI_DATA_PATH,"").sub(File.dirname(GFDNAVI_USER_PATH),"")
  rpath = '/' if rpath == ''   # local root directory
  dir = Directory.find(:first, :conditions=>["path=?",rpath], :user=>:all)
  if dir
    if OPTS[:force] or t_comp(mtime, dir.mtime)
#      dir.destroy   # for test
#      dir = nil     # for test
    elsif OPTS[:attr]
      parse_meta(path, dir.node)
      dir.save
      MSG.normal(".",".")
      return [dir, false]
    else
      MSG.normal(".",".")
      return [dir, false]
    end
    MSG.normal("\nupdating  #{rpath}",".")
  else
    MSG.normal("\nregistering  #{rpath}",".")
  end
  KeywordAttribute.transaction do
    dir ||= Directory.new
    if parent
      dir.name = File.basename(rpath)
    else
      dir.name = rpath
    end
    dir.path = rpath
    dir.mtime = mtime
    dir.size = size if size
    dir.plain_file = plain_file
    if parent
      dir.parent = Node===parent ? parent : parent.node
      dir.owner = parent.owner
    else
      dir.owner = ROOT
    end
    parse_meta(path, dir.node)
    kas.each{|k,v| register_kattr(k, v, dir.node)} if kas
    unless dir.save
      warn "failed to register directory(#{rpath})"
      warn "  #{dir.errors.full_messages}"
    end
    MSG.verbose("\nregister directory: #{rpath}")
  end
  return [dir, true]
end

def parse_meta(path,node)
  fname = path + ".yml"
  if File.exist?(fname)
    hash = YAML.load( File.read(fname) )
    if Hash === hash
      register_meta(hash,node)
    else
      warn "yaml file must have hash data"
    end
  end

  fname = path + ".SIGEN"
  if File.exist?(fname)
    File.foreach(fname){|line|
      k, v = line.chop.split(":")
      register_kattr(k, v.strip, node) if v
      
    }
  end
end

def register_meta(meta,node)
  meta.sort.each{|k,v| # sort to registe 'contains' firstly
    case k
    when "gfdnavi", "gfdnavi_knowledge"
      register_gfdnavi_params(v, node)
    when "contains"
      v.each{ |name,meta|
        case name
        when /^[^\/]*$/   # variables directory under node
          Var_Meta_To_Register[File.join(node.path,name)] = meta
        end
      }
    else
      register_kattr(k, v, node)
    end
  }
end

def register_kattr(name, value, node)
  return if value.nil?
  value=NArray[value] if value.is_a?(Numeric)  # to rescue a inapropriate yaml
  ka = nil
  node.keyword_attributes.each{ |nka|   # node unsaved, so find explicitly
    if nka.name == name
      ka = nka
      break
    end
  }
  unless ka
    ka = KeywordAttribute.new
    ka.name = Kconv.kconv(name, @charset)
    ka.node = Node === node ? node : node.node
  end
  ka.value = String===value ? Kconv.kconv(value, @charset) : value
  node.keyword_attributes.push(ka)  # --> ka will be automtcly saved with node
end

def register_gfdnavi_params(hash, node)
  node = node.node unless Node === node
  hash.each do |key, val|
    case key
    when 'aggregate'
      raise('aggregate must have Hash') unless Hash === val
      val.each{|name, ha|
        dname = File.dirname(name)
        vname = File.basename(name)
        fname = File.join(node.path, dname)
        if ha && ali = ha['alias']
          path = File.expand_path(File.join(node.path, ali))
          path.sub!(/^[a-zA-Z]:/,"")
          parent = Node.find(:first, :conditions=>["path=?",File.dirname(path)])
          parent
          raise("alias is invalid") unless parent
        else
          path = fname
          parent = node
        end
        ka = {'remark' => 'This is a vertual united file'}
        parent, flag = register_dir(path, parent, nil, nil, false, ka)
        register_var(fname, File.join(path,vname), parent, nil) if flag
      }
    when 'owner','user'   # 'user' is for backward compatibility
      user = User.find_by_login(val)
      if user
        node.owner = user
      end
    when 'other_mode'
      node.other_mode = val == 4 ? 4 : 0
    when 'rgroups'
      raise("Array expected") if !val.is_a?(Array)
      node.set_rgroups(*val)
    when 'wgroups'
      raise("Array expected") if !val.is_a?(Array)
      node.set_wgroups(*val)
    when 'downloadable'
      if node.directory?
        node.entity.downloadable = val==true
      end
    when 'draw_parameters'
      val.each{|dpk, dpv|
        dp = DrawParameter.new
        dp.node = node
        dp.name = dpk
        dp.value = dpv
        node.draw_parameters.push(dp)
      }
    when 'vizshot'
      if node.image?
        node.entity.vizshot = val
      else
        raise "gfdinavi attribute of 'vizshot' is only for Image"
      end
    when "references"
      if Array === val
        val.each{|hash|
          if Hash === hash
            @references.push [hash["name"], hash["path"], node.path]
          else
            raise "gfdnavi attribute of references must be Array of Hash('name' and 'path')"
          end
        }
      else
        raise "gfdnavi attribute of references must be Array of Hash('name' and 'path')"
      end
    when 'category'
      node.entity.category = val
      register_kattr("category", val, node)
    when 'title'
      node.entity.title = val
      register_kattr("title", val, node)
    when 'creator'
      node.entity.creator = val
      register_kattr("creator", val, node)
    when 'textbody'
      node.entity.textbody = val
      register_kattr("textbody", val, node)
    when 'description'
      node.entity.description = val
      register_kattr("description", val, node)
    when 'horizontal_figures'
      node.entity.horizontal_figures = val
    when 'default_layout'
      node.entity.default_layout = val
    when 'figures_size_status'
      node.entity.figures_size_status = val
    when 'figures_size_number'
      node.entity.figures_size_number = val
    when 'comment_on'
      if knowledge = Knowledge.find(:first, :conditions => ["path=?", val])
        node.entity.comment_on = knowledge.node_id
      else
        ToDo[val] = Proc.new {|node|
          knowledge.comment_on = Knowledge.find(node.id).id
          knowledge.save
        }
      end
    when 'comment_number'
      node.entity.comment_number = val
    when 'knowledge_figures'
      knowledge_figures = Array.new
      val.each do |v|
        kf = KnowledgeFigure.new
        kf.caption = v["caption"]
        if image = Image.find(:first, :conditions => ["path=?", v["image_path"]])
          kf.image = image
        else # 文書に入れるべき絵が未登録なら、ToDoリストに入れて後回し
          ToDo[v["image_path"]] = Proc.new {|node|
            image = Image.find(:first, :conditions => ["node_id=?", node.id])
            kf.image = image
            kf.save
          }
        end
        knowledge_figures.push(kf)
      end
      node.entity.knowledge_figures = knowledge_figures
    end
  end
end

##########################################
######         main part           #######
##########################################

@charset = Kconv::UTF8

#< interpret options >

opt = OptionParser.new
OPTS = {}
ARGV.options{|opt|
  opt.on( '-q', '--quiet', "Quiet mode" ){|v| OPTS[:quiet] = v}
  opt.on( '-v', '--verbose', "Verbose mode" ){|v| OPTS[:verbose] = v}
  opt.on( '-l', '--local-only', "Update only for local data" 
         ){|v| OPTS[:localonly] = v}
  opt.on( '-d=DIR', '--dir=DIR', "Process only under the directory"
         ){|v| OPTS[:dir] = v.sub(/^=/, '')}
  opt.on( '-f', '--force', "Force to (re-)register" ){|v| OPTS[:force] = v}
  opt.on( '-a', '--update-attributes', "Update attributes from YAML and SIGEN file" ){|v| OPTS[:attr] = v}
  opt.on( '--ignore-errors', "Ignore errors while opning files"){|v| OPTS[:ignore_errors] = v}
  opt.on( '--clear-tree', "Remove all nodes from DB. (Only to clear. No registering.)" ){|v| OPTS[:cleartree] = v}
  opt.on( '--st-remote', "Register space-time attributes of remote data as well as local data (Very slow with opendap. Be patient.)" ){|v| OPTS[:stremote] = v}
  ##See the top of this file##  opt.on( '-p', '--production', "Production DB" ){|v| OPTS[:production]=v}

  opt.on_tail('-h', '--help', "Show this help message"){|v| OPTS[:help] = v}
  opt.parse!
}

if OPTS[:help]
  print <<-"EOS"

  USAGE: ruby #{File.basename($0.to_s)} [options]

  OPTIONS: \n#{opt.to_a[1..-1].join("")}
  EOS
  exit
end

if OPTS[:quiet]
  MSG.to_quiet
elsif OPTS[:verbose]
  MSG.to_verbose
end

#< optional : remove all nodes from DB >

if OPTS[:cleartree]
  # Remove all root dirs. Children will be deleted automatically.
  rootdirs = Node.find(:all, :user=>:all, 
               :conditions=>'parent_id is NULL AND node_type = 0')
  print "Removing\n"
  rootdirs.each{|n| print("  ",n.path,"\n"); n.destroy}
  exit
end

#< register local data >

ROOT = User.find_by_login("root")
raise("Cannot find the user 'root'. Need to create it first.") if ROOT.nil?

begin

  @references = Array.new

  if !OPTS[:dir]

    rootdir = parse_dir(open_dir(GFDNAVI_DATA_PATH), nil, 
                        File.mtime(GFDNAVI_DATA_PATH))

    parse_dir(open_dir(GFDNAVI_USER_PATH), rootdir, File.mtime(GFDNAVI_USER_PATH))

    #< register remote data >

    if !OPTS[:localonly] && GFDNAVI_REMOTE_DATA_PATHS
      GFDNAVI_REMOTE_DATA_PATHS.each do |path|
        opddir = open_dir(path)
        parse_dir(opddir)
      end
    end

  else

    dirname = OPTS[:dir]
    dir = Node.find_by_path(dirname)  or  raise("#{dirname} is not in DB")
    parent = dir.parent  or  raise("parent of #{dirname} is not in DB")
    parse_dir( open_dir(dir.fname), parent, File.mtime(dir.fname) )

  end

  print "\nregistering node relations\n"
  @references.each{|ref|
    nr = NodeRelation.new
    nr.name = ref[0]
    nr.reference = Node.find(:first,:conditions=>["path=?",ref[1]], :user=>:all)
    nr.referenced_by = Node.find(:first,:conditions=>["path=?",ref[2]], :user=>:all)
    nr.save!
  }

rescue
  print "\n"
  raise $!
end

print "\n"