#!/usr/bin/env ruby
#:mode=ruby:

# This program is released under the GNU General Public Licence. Please see
# http://opensource.org/licenses/gpl-license.php for more information.
# Author: Doga Armangil, armangild@yahoo.com

require 'rexml/document'
require 'uri'
require 'open-uri'
require 'ostruct'
require 'optparse'
require 'pathname'
require 'date'
require 'cgi'

include REXML

PROGRAM_VERSION = "1.3.3"

#todo: resuming of failed media downloads
#todo: strategies for downloading shows in chronological order (useful for audiobooks etc)
#todo: --subscriptions option (subscription d/l limit)
#todo: informative exception messages
#todo: only fetch bittorrent metainfo for d/l candidates
#todo: option to download shows concurrently
#todo: "lock" directory to prevent concurrency issues
#todo: option to throttle non-bittorrent downloads
#todo: cookie repository
# 1.3.3: added the -p option that assigns a separate cache subfolder to each feed
# 1.3.2: bug fix
# 1.3.1: added robust handling of subscription lists that directly link to media files (such links are now ignored), fixed an OPML generation bug for interrupted searches
# 1.3.0: added search function for online podcast directories such as the iPodder podcast directory, added xspf support
# 1.2.0: added support for decentralized subscription lists (i.e. subscription lists that point to other subscription lists), fixed a bug that sometimes caused an invalid Referer header to be sent in HTTP requests, added the -f option, added support for Atom feeds that do not list items in reverse chronological order, added support for RSS/Atom feeds as command line arguments, added support for Extended M3U and Extended PLS playlist formats, M3U playlists can now also be generated in vodcatcher mode, m3u is now the default type in vodcatcher mode, added "cache" strategy which deprecates -c option
# 1.1.1: added support for iTunes .pcast subscription files
# 1.1.0: names of media files downloaded via BitTorrent are now preserved, done some refactoring so that the script can function as a vodcatcher
# 1.0.4: added support for RSS feeds that do not list items in reverse chronological order
# 1.0.3: fixed an RSS parsing bug that caused enclosures of some feeds to be ignored
# 1.0.2: fixed some minor MP3 file naming bugs
# 1.0.1: names of downloaded MP3 files are now preserved
# 1.0.0: added ATOM support
# 0.4.0: added duplicate removal for MP3, RSS/Atom and OPML URLs and pathnames; added the -i option that attempts to increase the listen-time given to podcasts which frequently release short shows
# 0.3.2: fixed BitTorrent handling bug
# 0.3.1: added robust handling of network exceptions, removed support for Ctrl-C to terminate execution
# 0.3.0: added support for opml format used by podcastalley, added podcast title information in playlists, reduced RAM usage by not loading the history file in memory, history file and playlist are now updated after each download
# 0.2.1: added support for Ctrl-C to terminate execution; added robust handling of some bad command line arguments; (James Carter patch) fixed the "OPML truncation" issue where a bad RSS feed was considered the last of the list
# 0.2.0: added a new download strategy ("one"); added support for more than one OPML argument, fixed some issues
# 0.1.7: bug fix
# 0.1.6: added internal Bittorrent support, fixed flawed handling of some exceptions
# 0.1.5: changed -d option description, added external handling of Bittorrent files
# 0.1.4: bug-fix, robust handling of bad //enclosure/@length attributes, handling of relative enclosure URLs
# 0.1.3: podcast download strategies (and changed default), download retries
# 0.1.2: added TOX playlist support, added HTTP and FTP support for the OPML parameter, done some code clean-up
# 0.1.1: fixed RSS parsing issue
# 0.1.0: initial version

opt = OpenStruct.new

opt.MODES = [:pod, :vod]
opt.mode = opt.MODES[0] #podcatcher mode
opt.mode = opt.MODES[1] #vodcatcher mode

opt.PLAYLIST_TYPES = [:m3u, :smil, :pls, :asx, :tox, :xspf]
opt.playlist_type = opt.PLAYLIST_TYPES[0]

opt.size = case opt.mode
when :pod
	150
when :vod
	500
end 
opt.USER_AGENT = "#{opt.mode}catcher/#{PROGRAM_VERSION} Ruby/#{RUBY_VERSION}"

opt.content_type = case opt.mode
	when :pod
		Regexp.new '^audio/mpeg$'
	when :vod
		Regexp.new '^video/'
	end 

opt.DESCRIPTION = case opt.mode
when :pod
<<END

Armangil's podcatcher is a podcast client for the command line. It provides 
several download strategies, supports BitTorrent, offers cache management, 
and generates playlists for MP3 player applications. It accepts subscription 
documents (in OPML or iTunes PCAST format) or feed documents (RSS or Atom)
as argument. Also included: a search function for subscribing to feeds.

BitTorrent podcasts are supported both internally (through the RubyTorrent
library) and externally (.torrent files are downloaded, but the user handles
them using another BitTorrent application). The latter is currently the most
reliable method, as RubyTorrent is still in alpha phase.

Concurrency is not handled: simultaneous executions of this program should
target different directories.

See http://en.wikipedia.org/wiki/Podcasting for more information.

Usage1: #{$0} [options] subscription-or-feed-url-or-filename+ 
Usage2: #{$0} [options] (subscription or feed from standard input) 
Usage3: #{$0} -F search [options] query subscription-or-feed-url-or-filename+
Usage4: #{$0} -F search [options] query (subscription from standard input) 
END
when :vod
<<END

Armangil's vodcatcher is a vodcast client for the command line. It provides 
several download strategies, supports BitTorrent, offers cache management, 
and generates playlists for video player applications. It accepts subscription 
documents (in OPML or iTunes PCAST format) or feed documents (RSS or Atom)
as argument. Also included: a search function for subscribing to feeds.

BitTorrent vodcasts are supported both internally (through the RubyTorrent
library) and externally (.torrent files are downloaded, but the user handles
them using another BitTorrent application). The latter is currently the most
reliable method, as RubyTorrent is still in alpha phase.

Concurrency is not handled: simultaneous executions of this program should
target different directories.                                               

See http://en.wikipedia.org/wiki/Vodcast for more information.

Usage1: #{$0} [options] subscription-or-feed-url-or-filename+ 
Usage2: #{$0} [options] (subscription or feed from standard input) 
Usage3: #{$0} -F search [options] query subscription-or-feed-url-or-filename+
Usage4: #{$0} -F search [options] query (subscription from standard input) 
END
end
opt.dir = Pathname.new Dir.pwd
opt.memsize = 1_000
opt.empty = false
opt.simulate = false
opt.verbose = false
opt.STRATEGIES = [:new, :one, :back_catalog, :all, :cache]
opt.strategy = opt.STRATEGIES[0]
opt.retries = 1
opt.torrent_dir = nil
opt.rubytorrent = false
opt.upload_rate = nil #10
opt.itemsize = 0
opt.feeds = 1_000
opt.FUNCTIONS = [:download, :search]
opt.function = opt.FUNCTIONS[0]
opt.per_feed = false

OptionParser.new(){|c|
	c.banner = opt.DESCRIPTION
	c.separator ""
	c.separator "Options:"
	tmp = opt.FUNCTIONS.collect(){|e| "#{e} "}
	c.on("-F", "--Function FUNCTION", opt.FUNCTIONS, "used function (#{tmp});", 
		"when using the search function (which", "generates an OPML document), the only", 
		"relevant options are -v, -r and -f",
		"(default value: #{opt.function})\n"){|e|
		opt.function = e if e
	}
	c.on("-d", "--dir DIR", "Query for caching files", "and storing application state", "(default value: current directory)\n"){|e|
		opt.dir = opt.dir+e
		opt.dir.mkdir unless opt.dir.exist? 
		exit 1 unless opt.dir.directory?
	}
	c.on("-s", "--size SIZE", "Size of media file cache in MB; ", "0 means infinite; default value is #{opt.size}\n"){|e|
		opt.size = e.to_i
		opt.size = nil if opt.size<1
	}
	c.on("-m", "--memsize N", "Remember last N downloaded media files,",
		"and do not download them again; ", 
		"0 means infinite; default value is #{opt.memsize}\n"){|e|
		opt.memsize = e.to_i
		opt.memsize = nil if opt.memsize<1
	}
	tmp = opt.PLAYLIST_TYPES.collect(){|e| "#{e} "}
	c.on("-t", "--type TYPE", opt.PLAYLIST_TYPES, "Playlist type (#{tmp}) ",
		"(default value: #{opt.playlist_type})\n"){|e|
		opt.playlist_type = e if e
	}
	tmp = opt.STRATEGIES.collect(){|e| "#{e} "}
	c.on("-S", "--strategy S", opt.STRATEGIES, 
		"Strategy to use for downloading #{opt.mode}casts",
		"(#{tmp}):",
		"* back_catalog: download any show that",
		"has not been downloaded before; prefer",
		"the most recent shows of top-most feeds",
		"in the OPML/PCAST documents.",
		"* new: ignore #{opt.mode}cast back-catalogs; this ",
		"is like back_catalog, except that only the",
		"most recent show of each feed is ",
		"considered.",
		"* one: download one unlistened show of",
		"each feed, with a preference for recent",
		"shows",
		"* all: download all show; even already",
		"downloaded shows are downloaded once ",
		"again. ",
		"* cache: play shows that are already in",
		"cache (default value: #{opt.strategy})\n"){|e|
		opt.strategy = e if e
	}
	c.on("-f", "--feeds N", 
		"Do not download more than N feeds", "(when using the download function);",
		"or return the first N relevant feeds", "(when using the search function);",
		"0 means no limits (default value: #{opt.feeds})\n"){|e|
		opt.feeds = e.to_i
		opt.feeds = nil if opt.feeds<1
	}
	c.on("-r", "--retries N", 
		"Try downloading files (media, RSS/Atom or",
		"or OPML/PCAST) at most N times before", 
		"giving up (default value: #{opt.retries})\n"){|e|
		opt.retries = e.to_i unless e.to_i<1
	}
	c.on("-T", "--Torrentdir DIR", "Copy torrent files to directory DIR;",
		"handling torrents through an external ","BitTorrent client is left to the user;  ", 
		"if this option is not used, torrents are ","handled internally (if RubyTorrent is ","installed), or else ignored\n"){|e|
		opt.torrent_dir = Pathname.new e
		opt.torrent_dir = nil unless opt.torrent_dir.exist? and opt.torrent_dir.directory?
	}
	c.on("-U", "--Uploadrate N", 
		"Maximum upload rate (kilobytes per second)",
		"for the internal BitTorrent client ",
		"(default: #{opt.upload_rate ? opt.upload_rate : 'no limit'})\n"){|e|
		opt.upload_rate = e.to_i unless e.to_i<1
	}
	c.on("-i", "--itemsize N", 
		"If a downloaded show is less than N MB in",
		"size (where N is an integer), fetch other",
		"shows of that same feed until this size is",
		"reached (default: #{opt.itemsize})\n"){|e|
		opt.itemsize = e.to_i unless e.to_i<0
	}
	c.on("-p", "--[no-]perfeed", "Create one subfolder per feed in cache\n"){|e|
		opt.per_feed = e
	}
	c.on("-c", "--[no-]cache", "Play already cached files,", "DEPRECATED (use -S cache)\n"){|e|
		opt.strategy = :cache if e
	}
	c.on("-e", "--[no-]empty", "Delete all previously downloaded ", "files\n"){|e|
		opt.empty = e
	}
	c.on("-a", "--[no-]asif", "Do not download media files", "(useful for testing)\n"){|e|
		opt.simulate = e
	}
	c.on("-v", "--[no-]verbose", "Run verbosely\n"){|e|
		opt.verbose = e
	}
	c.on("--version", "Display current version\n"){
		puts PROGRAM_VERSION
		exit
	}
	c.on("-h", "--help", "Show this message\n"){
		puts c
		exit
	}
	c.separator ""
	c.separator "Usage examples:"
	case opt.mode
	when :pod
		c.separator "    #{$0} -d ~/podcasts http://podcastalley.com/PodcastAlleyTop50.opml > latest.m3u"
		c.separator ""
		c.separator "    #{$0} -d ~/podcasts rss.xml atom.xml *.pcast feeds.opml http://host/second.opml > latest.m3u"
		c.separator ""
		c.separator "    #{$0} -d ~/podcasts -S cache > cache.m3u"
		c.separator ""
		c.separator "    cat feeds.opml | #{$0} -d ~/podcasts > latest.m3u"
		c.separator ""
		c.separator "    #{$0} -vd ~/podcasts -s 500 -m 10_000 -t tox feeds.opml > latest.tox"
		c.separator ""
		c.separator "    #{$0} -vF search 'book health daily source code' 'http://www.ipodder.org/discuss/reader$4.opml' > results.opml"
		c.separator ""
		c.separator "    #{$0} -F search -f 12 mac http://www.podfeed.net/opml/directory.opml > results.opml"
	when :vod
		c.separator "    #{$0} -d ~/vodcasts feeds.opml http://host/second.opml > latest.m3u"
		c.separator ""
		c.separator "    #{$0} -d ~/vodcasts -S cache > cache.m3u"
		c.separator ""
		c.separator "    cat feeds.opml | #{$0} -d ~/vodcasts > latest.m3u"
		c.separator ""
		c.separator "    #{$0} -vd ~/vodcasts -s 500 -m 10_000 -t asx feeds.opml > latest.asx"
		c.separator ""
		c.separator "    #{$0} -vF search -f 44 'video vlog' http://www.podfeed.net/opml/directory.opml > results.opml"
	end 
}.parse! 

opt.size *= 1_000_000 if opt.size
opt.upload_rate *= 1024 if opt.upload_rate
opt.itemsize *= 1_000_000 

begin 
	require "rubytorrent"
	opt.rubytorrent = true
	$stderr.puts "RubyTorrent detected" if opt.verbose
rescue Interrupt, SystemExit
	exit 1
rescue Exception
end 

class Playlist
	def initialize(playlisttype)
		@playlisttype = playlisttype
		@size = 0
	end
	def start()
		@str = ""
		case @playlisttype
		when :tox
			@str = "# toxine playlist \n"
		when :m3u
			@str = "#EXTM3U\n"
		when :pls
			@str = "[playlist]\n"
		when :asx
			@str = <<END
<asx version = "3.0">
END
		when :smil
			@str = <<END
<?xml version="1.0"?>
<!DOCTYPE smil PUBLIC "-//W3C//DTD SMIL 2.0//EN" "http://www.w3.org/2001/SMIL20/SMIL20.dtd">
<smil xmlns="http://www.w3.org/2001/SMIL20/Language">
  <head></head>
  <body>
END
		when :xspf
			@doc = Document.new
			@doc.xml_decl.dowrite
			@doc.add_element Element.new("playlist")
			@doc.root.add_attribute "version", "1"
			@doc.root.add_attribute "xmlns", "http://xspf.org/ns/0/"
			@tracklist = Element.new("trackList")
			@doc.root.add_element @tracklist
		end
		print @str
		@str
	end 
	def add(content)
		@size+=1
		entry = ""
		case @playlisttype
		when :m3u
			entry = "#EXTINF:-1,[#{content.feed_title and content.feed_title.gsub(/,/," ")}] #{content.title and content.title.gsub(/,/," ")}\n#{content.file.to_s}\n"
		when :pls
			entry = "File#{@size}:#{content.file}\nTitle#{@size}:[#{content.feed_title}] #{content.title}\nLength#{@size}:-1\n"
		when :asx
			entry = "  <entry><ref href='#{content.file.to_s.gsub(/&/,"&amp;").gsub(/'/,"&apos;").gsub(/"/,"&quot;")}' /></entry>\n"
		when :smil
			entry = "  <ref src='#{content.file.to_s.gsub(/&/,"&amp;").gsub(/'/,"&apos;").gsub(/"/,"&quot;")}' />\n"
		when :tox
			entry = "entry { \n\tidentifier = [#{content.feed_title}] #{content.title};\n\tmrl = #{content.file};\n};\n"
		when :xspf
			track = Element.new("track")
			@tracklist.add_element track
			title = Element.new("title")
			title.add_text "[#{content.feed_title}] #{content.title}"
			track.add_element title
			location = Element.new("location")
			location.add_text fileurl(content.file) 
			track.add_element location
		end
		@str += entry
		print entry
		entry
	end 
	def finish()
		res = ""
		case @playlisttype
		when :tox
			res = "# end "
		when :asx
			res = <<END
</asx>
END
		when :smil
			res = <<END
  </body>
</smil>
END
		when :pls
			res = "NumberOfEntries=#{@size}\nVersion=2\n"
		when :xspf
			@doc.write $stdout, 0
		end 
		@str += res
		print res
		res
	end 
	def to_s()
		if @doc
			@doc.to_s
		else 
			@str
		end
	end 
private
	def fileurl(path)
		res = ""
		loop {
			path, base = path.split
			if base.root?
				if base.to_s != "/"
					res = "/"+CGI.escape(base.to_s)+res
				end
				break
			end
			res = "/"+CGI.escape(base.to_s)+res
		}
		"file://"+res
	end 
end

class History
	def initialize(dir)
		@history = dir + "history"
		@history_old = dir + "history-old"
		unless @history.exist?
			@history_old.rename @history if @history_old.exist?
		end
		@history.open("w"){|f|}unless @history.exist?
		exit 1 unless @history.file?
		@history_old.delete if @history_old.exist?
	end
	def markoldshows(feeds)
		feeds.each(){|feed|
			feed.each(){|content|
				content.in_history = false
			}
		}
		@history.each_line(){|url|
			url = url.chomp
			feeds.each(){|feed|
				feed.each(){|content|
					next if content.in_history
					content.in_history = content.url == url
				}
			}
		}
	end 
	def add(content)
		begin 
			@history.open("a"){|f|
				f.puts content.url
			}
		rescue Interrupt, SystemExit
			exit 1
		rescue Exception
			$stderr.puts "Error: history file could not be updated"
		end
	end 
	def trim(limit)
		begin 
			history_size = 0
			@history.each_line(){|url|
				history_size += 1
			}
			if history_size > limit #shrink
				@history_old.delete if @history_old.exist?
				@history.rename @history_old
				@history.open("w"){|f|
					@history_old.each_line(){|url|
						f.print(url) if history_size <= limit
						history_size -= 1
					}
				}
				@history_old.unlink
			end
		rescue Interrupt, SystemExit
			exit 1
		rescue Exception
			$stderr.puts "Error during history file clean-up."
		end if limit
	end 
end 

class Cache 
	def initialize(opt)
		super()
		@opt = opt
		@@TORRENT = "application/x-bittorrent"
		@@ATOM_NS = Regexp.new "^http://purl.org/atom/ns#"
		#history
		@history = History.new opt.dir 
		#cache 
		@cache_dir = opt.dir+"cache"
		@cache_dir.mkdir() unless @cache_dir.exist?
		exit 1 unless @cache_dir.directory?
		@cache_dir.each_entry(){|e|
			e = @cache_dir+e
			e = e.cleanpath
			next if e == @cache_dir or e == opt.dir
			if e.directory? #feed subfolder
				e.each_entry(){|e2|
					e2 = e+e2
					next if e2.directory?
					if opt.empty
						$stderr.puts "deleting: #{e2}" if opt.verbose
						e2.delete
					end
				}
				e.delete if e.entries.size == 2
			elsif opt.empty
				$stderr.puts "deleting: #{e}" if opt.verbose
				e.delete
			end
		}
		@cache = @cache_dir.entries.collect(){|e|
			e = @cache_dir+e
			e = e.cleanpath
			next if e == @cache_dir or e == opt.dir
			if e.file?
				content = OpenStruct.new
				content.file = e
				content.size = e.size
				content.title = e.to_s
				content
			elsif e.directory?
				e.entries.collect(){|e2|
					e2 = e+e2
					if e2.file?
						content = OpenStruct.new
						content.file = e2
						content.size = e2.size
						content.title = e2.to_s
						content
					else 
						nil
					end
				}
			else 
				nil
			end
		}
		@cache.flatten!
		@cache.compact!
		@cache.sort!(){|e,e2|
			e.file.mtime() <=> e2.file.mtime()
		}
	end 
	def createplaylist(urls) 
		playlist = Playlist.new @opt.playlist_type
		if @opt.strategy == :cache
			playlist.start
			@cache.reverse!
			@cache.each(){|content|
				playlist.add content
			}
			playlist.finish
			return playlist.to_s
		end
		playlist.start
		doc = nil
		if urls.size == 0
			$stderr.puts "Reading subscriptions/feed from standard input" if @opt.verbose
			begin 
				xml = ""
				$stdin.each(){|e|
					xml += e
				}
				doc = OpenStruct.new
				doc.dom = Document.new(xml)
				doc = nil unless doc.dom
			rescue Interrupt, SystemExit
				exit 1
			rescue Exception
				$stderr.puts "Error: unreadable subscriptions/feed"
				doc = nil
			end 
		end
		dochistory = []
		feeds = []
		urls.uniq!
		links = urls.collect(){|e|
			l = OpenStruct.new
			l.url = e
			l
		}
		loop {	
			break if @opt.feeds and feeds.size >= @opt.feeds
			while not doc
				link = links.shift
				break unless link
				if dochistory.detect{|e| e == link.url}
					$stderr.puts "Skipping duplicate: #{link.url}" if @opt.verbose
					next
				end
				$stderr.puts "Fetching subscriptions/feed: #{link.url}" if @opt.verbose
				dochistory << link.url
				begin 
					doc = fetchdoc(link)
				rescue Interrupt, SystemExit
					exit 1
				rescue Exception
					$stderr.puts "Error: skipping unreadable subscriptions/feed"
				end 
			end
			break unless doc
			begin
				if doc.dom.root.name == "opml"
					newlinks = []
					XPath.each(doc.dom,"//outline"){|outline|
						url = outline.attributes["xmlUrl"]
						url = outline.attributes["url"] unless url
						next unless url
						begin 
							url = URI.parse(doc.url).merge(url).to_s if doc.url
						rescue Interrupt, SystemExit
							exit 1
						rescue Exception
						end
						link = OpenStruct.new
						link.url = url
						link.referrer = doc.url
						newlinks << link
					}
					links = newlinks + links
				elsif doc.dom.root.name == "pcast"
					newlinks = []
					XPath.each(doc.dom,"//link[@rel='feed']"){|outline|
						url = outline.attributes["href"]
						next unless url
						begin 
							url = URI.parse(doc.url).merge(url).to_s if doc.url
						rescue Interrupt, SystemExit
							exit 1
						rescue Exception
						end
						link = OpenStruct.new
						link.url = url
						link.referrer = doc.url
						newlinks << link
					}
					links = newlinks + links
				elsif doc.dom.root.namespace =~ @@ATOM_NS
					feed = []
					XPath.each(doc.dom.root,"//*[@rel='enclosure']"){|e2|
						next unless e2.namespace =~ @@ATOM_NS
						content = OpenStruct.new
						XPath.each(e2,"parent::/title/text()"){|node|
							content.title = ""
							node.value.each(){|e3| #remove line breaks
								content.title+= e3.chomp+" "
							}
							content.title.strip!
						}
						XPath.each(e2,"parent::/created/text()"){|node|
							pub_date = ""
							node.value.each(){|e3| #remove line breaks
								pub_date+= e3.chomp+" "
							}
							begin 
								content.pub_date = DateTime.parse(pub_date.strip, true)
							rescue Exception
							end 
						}
						content.mime = e2.attributes["type"].downcase
						next if @opt.content_type !~ content.mime and content.mime != @@TORRENT
						next if content.mime == @@TORRENT and not (@opt.torrent_dir or @opt.rubytorrent)
						content.feedurl = doc.url
						content.url = URI.parse(content.feedurl).merge(e2.attributes["href"]).to_s if content.feedurl
						content.size = e2.attributes["length"].to_i
						content.size = 0 if content.mime == @@TORRENT #not strictly necessary
						feed << content
					}
					#sort by date
					feed.sort!(){|a,b|
						if a.pub_date
							if b.pub_date
								b.pub_date <=> a.pub_date
							else 
								-1
							end
						else 
							if b.pub_date
								1
							else 
								0
							end
						end
					}
					feed.each(){|content|
						$stderr.puts "Enclosure: #{content.url}"
					} if @opt.verbose
					#title
					node = XPath.first(doc.dom,"/feed/title/text()")
					feed_title = ""
					node.value.each(){|e3| #remove line breaks
						feed_title += e3.chomp+" "
					}
					feed_title.strip!
					feed.each(){|content|
						content.feed_title = feed_title
					}
					#
					feeds << feed
				elsif doc.dom.root.name = "rss"
					feed = []
					doc.dom.root.elements.each(){|e|
						e.elements.each(){|e1|
							e1.elements.each(){|e2|
								if e2.name == "enclosure"
									content = OpenStruct.new
									XPath.each(e2,"parent::/title/text()"){|node|
										content.title = ""
										node.value.each(){|e3| #remove line breaks
											content.title+= e3.chomp+" "
										}
										content.title.strip!
									}
									XPath.each(e2,"parent::/pubDate/text()"){|node|
										pub_date = ""
										node.value.each(){|e3| #remove line breaks
											pub_date+= e3.chomp+" "
										}
										begin 
											content.pub_date = DateTime.parse(pub_date.strip, true)
										rescue Exception
										end 
									}
									content.mime = e2.attributes["type"].downcase
									next if @opt.content_type !~ content.mime and content.mime != @@TORRENT
									next if content.mime == @@TORRENT and not (@opt.torrent_dir or @opt.rubytorrent)
									content.feedurl = doc.url
									content.url = URI.parse(content.feedurl).merge(e2.attributes["url"]).to_s if content.feedurl
									content.size = e2.attributes["length"].to_i
									content.size = 0 if content.mime == @@TORRENT #not strictly necessary
									feed << content
								end
							}if e1.name == "item"
						}if e.name == "channel"
					}
					#sort by date
					feed.sort!(){|a,b|
						if a.pub_date
							if b.pub_date
								b.pub_date <=> a.pub_date
							else 
								-1
							end
						else 
							if b.pub_date
								1
							else 
								0
							end
						end
					}
					feed.each(){|content|
						$stderr.puts "Enclosure: #{content.url}"
					} if @opt.verbose
					#title
					node = XPath.first(doc.dom,"//channel/title/text()")
					feed_title = ""
					node.value.each(){|e3| #remove line breaks
						feed_title += e3.chomp+" "
					}
					feed_title.strip!
					feed.each(){|content|
						content.feed_title = feed_title
					}
					#
					feeds << feed
				end
			rescue Interrupt, SystemExit
				exit 1
			rescue Exception
				$stderr.puts "Error: skipping unreadable subscriptions/feed document"
			end
			doc = nil
		}
		#apply download strategy
		@history.markoldshows feeds
		feeds.each(){|feed|
			in_hist = false
			itemsize = sum = 0
			for i in 0...feed.size
				in_hist = (in_hist or feed[i].in_history)
				sum +=1 unless feed[i].in_history
				itemsize += feed[i].size
				next if @opt.strategy == :all
				next if @opt.strategy == :back_catalog and (not feed[i].in_history)
				next if @opt.strategy == :new and (not in_hist) and (itemsize < @opt.itemsize or i == 0)
				if @opt.strategy == :one
					feed[i] = nil unless sum == 1 and (not feed[i].in_history)
				else 
					feed[i] = nil
				end
			end
			feed.compact!
		}
		#remove duplicate shows
		feeds.each(){|feed|
			feed.each(){|content|
				next unless content
				dup = false
				feeds.each(){|f|
					for i in 0...f.size
						next unless f[i]
						if f[i].url == content.url
							f[i] = nil if dup
							dup = true
						end
						$stderr.puts "Removed duplicate: #{content.url}" unless f[i] or (not @opt.verbose)
					end
				}
			}
			feed.compact!
		}
		#fetch torrent metainfo files
		feeds.each(){|feed|
			feed.each(){|content|
				next if content.mime != @@TORRENT
				content.mime = nil
				begin 
					$stderr.puts "Fetching torrent metainfo: #{content.url}" if @opt.verbose
					content.metainfo = RubyTorrent::MetaInfo.from_location content.url
					content.size = content.metainfo.info.length
					content.mime = case content.metainfo.info.name.downcase
						when /\.mp3$/
							"audio/mpeg"
						when /\.wma$/
							"audio/x-ms-wma"
						when /\.mpg$|\.mpeg$|\.mpe$|\.mpa$|\.mp2$|\.mpv2$/
							"video/mpeg"
						when /\.mov$|\.qt$/
							"video/quicktime"
						when /\.avi$/
							"video/x-msvideo"
						when /\.wmv$/
							"video/x-ms-wmv"
						when /\.asf$/
							"video/x-ms-asf"
						when /\.mp4$|\.mpg4$/
							"video/mp4"
						else 
							nil
						end 
					content.url = nil unless content.mime
					content.url = nil unless (@opt.content_type =~ content.mime)
					content.url = nil unless content.metainfo.info.single?
				rescue Interrupt
					content.url = nil
					$stderr.puts "Error: unreadable torrent metainfo" if @opt.verbose
				rescue SystemExit
					exit 1
				rescue Exception
					content.url = nil
					$stderr.puts "Error: unreadable torrent metainfo" if @opt.verbose
				end 
			}
			for i in 0...feed.size
				feed[i] = nil unless feed[i].url
			end
			feed.compact!
		}
		#fetch enclosures
		item = total = 0
		@cache.each(){|e|
			total+= e.size
		}
		torrents = []
		torrentfiles = []
		begin
			inc = 0
			feeds.each {|e|
				#find next enclosure in feed
				content = e.shift
				next unless content
				#make place in cache
				while @opt.size and content.size+inc+total > @opt.size
					break if @opt.simulate
					f = @cache.shift
					break unless f
					total-= f.size
					parent = f.file.parent
					f.file.delete
					if parent.parent != @opt.dir and parent.entries.size == 2
						#delete empty feed subfolder
						parent.delete
					end
				end
				unless @opt.simulate
					break if @opt.size and content.size+inc+total > @opt.size
				end
				#download
				1.upto(@opt.retries){|i|
					begin 
						if content.metainfo
							if @opt.torrent_dir
								loop {
									content.file = @opt.torrent_dir+(Time.now.to_f.to_s+".torrent")
									break unless content.file.exist?
									sleep 1
								}
								$stderr.puts "Copying: #{content.url} to #{content.file}" if @opt.verbose and i == 1
								if not @opt.simulate 
									if content.feedurl and (content.feedurl =~ %r{^http:} or content.feedurl =~ %r{^ftp:})
										open(content.url, "User-Agent" => @opt.USER_AGENT, "Referer" => content.feedurl){|fin|
											content.file.open("wb"){|fout|
												fin.each_byte(){|b|
													fout.putc b
												}
											}
										}
									else 
										open(content.url, "User-Agent" => @opt.USER_AGENT){|fin|
											content.file.open("wb"){|fout|
												fin.each_byte(){|b|
													fout.putc b
												}
											}
										}
									end
								end
							else
								$stderr.puts "Fetching in background: #{content.url}" if @opt.verbose and i == 1
								unless @opt.simulate
									content.file = filename(content, @cache_dir)
									package = RubyTorrent::Package.new content.metainfo, content.file.to_s
									bt = RubyTorrent::BitTorrent.new content.metainfo, package, :dlratelim => nil, :ulratelim => @opt.upload_rate, :http_proxy => ENV["http_proxy"]
									torrents << bt
									torrentfiles << content
								end
								inc+= content.size
							end
						else 
							$stderr.puts "Fetching: #{content.url} (#{content.size.to_s} bytes)" if @opt.verbose and i == 1
							if not @opt.simulate 
								headers = {"User-Agent" => @opt.USER_AGENT}
								headers["Referer"] = content.feedurl if content.feedurl and (content.feedurl =~ %r{^http:} or content.feedurl =~ %r{^ftp:})
								if content.file
									headers["Range"] = "bytes=#{content.file.size}-"
								else 
									content.file = filename(content, @cache_dir)
									content.file.open("wb"){|fout|}
								end
								except = nil
								if content.accept_ranges
									content.file.open("ab"){|fout|
										begin 
											open(content.url, headers){|fin|
												fin.each_byte(){|b|
													fout.putc b
												}
											}
										rescue Exception
											#$stderr.puts "append error: #{$!}"
											except = $!
										end 
									}
								else 
									content.file.open("wb"){|fout|
										begin 
											open(content.url, headers){|fin|
												content.accept_ranges = fin.meta["accept-ranges"]
												fin.each_byte(){|b|
													fout.putc b
												}
											}
										rescue Interrupt, SystemExit, Exception
											#$stderr.puts "start error: #{$!.class} ranges:#{content}"
											except = $!
										end 
									}
								end
								raise except if except
							end
							content.size = content.file.size unless @opt.simulate
							playlist.add(content)
							@history.add content unless @opt.simulate
							inc+= content.size
						end
						break
					rescue Interrupt
					rescue SystemExit
						exit 1
					rescue Exception
					end 
					$stderr.puts "Attempt #{i} aborted"
					if content.file and i == @opt.retries
						if content.file.exist?
							parent = content.file.parent
							content.file.delete
							if parent.parent != @opt.dir and parent.entries.size == 2
								#delete empty feed subfolder
								parent.delete
							end
						end
						content.file = nil
					end
					sleep 5
				}
				redo if @opt.strategy == :new
			} 
			total+=inc
		end while inc>0
		#shut down torrents
		if torrents.length > 0
			$stderr.puts "Fetching torrents (duration: 30min to a couple of hours) " if @opt.verbose
			bt = torrents[0]
			completion = torrents.collect(){|e|
				e.percent_completed
			} 
			while torrents.length > 0
				sleep 30*60
				for i in 0...torrents.length
					c = torrents[i].percent_completed
					complete = torrents[i].complete? 
					$stderr.puts "Fetched: #{c}% of #{torrentfiles[i].url} " if @opt.verbose
					if complete or c == completion[i]
						begin 
							torrents[i].shutdown
						rescue SystemExit
							exit 1
						rescue Interrupt, Exception
						end 
						if complete
							playlist.add(torrentfiles[i])
							@history.add torrentfiles[i]
						else                                                        
							$stderr.puts "Aborted: #{torrentfiles[i].url}" if @opt.verbose
							begin 
								torrentfiles[i].file.delete if torrentfiles[i].file.exist?
								torrentfiles[i] = nil
							rescue Interrupt, SystemExit
								exit 1
							rescue Exception
							end 
						end
						torrents[i] = nil
						torrentfiles[i] = nil
						completion[i] = nil
						next
					end
					completion[i] = c
				end 
				torrents.compact!
				torrentfiles.compact!
				completion.compact!
			end
			begin 
				bt.shutdown_all
			rescue Interrupt, SystemExit
				exit 1
			rescue Exception
			end
			$stderr.puts "BitTorrent stopped" if @opt.verbose
		end
		playlist.finish
		@history.trim(@opt.memsize) unless @opt.simulate or @opt.strategy == :cache
		playlist.to_s
	end 
private
	def fetchdoc(link)
		doc = ""
		1.upto(@opt.retries){|i|
			begin 
				if link.url =~ %r{^http:} or link.url =~ %r{^ftp:}
					if link.referrer and (link.referrer =~ %r{^http:} or link.referrer =~ %r{^ftp:})
						open(link.url, "User-Agent" => @opt.USER_AGENT, "Referer" => link.referrer){|f|
							break if f.content_type.index "audio/"
							break if f.content_type.index "video/"
							f.each_line(){|e|
								doc += e
							}
						}
					else 
						open(link.url, "User-Agent" => @opt.USER_AGENT){|f|
							break if f.content_type.index "audio/"
							break if f.content_type.index "video/"
							f.each_line(){|e|
								doc += e
							}
						}
					end
				else 
					open(link.url){|f|
						f.each_line(){|e|
							doc += e
						}
					}
				end
				break
			rescue Interrupt
			rescue SystemExit
				exit 1
			rescue Exception
			end 
			$stderr.puts "Attempt #{i} aborted"
			doc = ""
			sleep 5
		}
		res = OpenStruct.new
		begin 
			res.dom = Document.new doc
		rescue Exception
		end 
		if res.dom
			res.url = link.url
		else 
			res = nil
		end
		res
	end 
	def filename(content, dir) #produce filename for content to be downloaded
		begin #per-feed subfolder
			if @opt.per_feed and content.feed_title and content.feed_title.size > 0
				newdir = dir+content.feed_title
				if newdir.exist?
					if newdir.directory?
						dir = newdir
					end
				else 
					newdir.mkdir
					dir = newdir
				end
			end
		rescue Exception
		#	$stderr.puts "error: #{$!}"
		end 
		ext = [""]
		if content.metainfo
			begin 
				ext = ["."+content.metainfo.info.name.split(".").reverse[0]]
			rescue Exception
			end 
		else 
			ext = case content.mime.downcase
				when "audio/mpeg"
					[".mp3"]
				when "audio/x-ms-wma"
					[".wma"]
				when "video/mpeg"
					[".mpg",".mpeg",".mpe",".mpa",".mp2",".mpv2"]
				when "video/quicktime"
					[".mov",".qt"]
				when "video/x-msvideo"
					[".avi"]
				when "video/x-ms-wmv"
					[".wmv"]
				when "video/x-ms-asf"
					[".asf"]
				when "video/mp4"
					[".mp4",".mpg4"]
				end 
		end
		#name from url?
		name = nil
		begin 
			if content.metainfo
				name = content.metainfo.info.name
				name = nil if (dir+name).exist?
			else 
				urlname = URI.split(content.url)[5].split("/")[-1]
				ext.each(){|e|
					if urlname[-e.length..-1].downcase == e
						name = urlname
						name = URI.unescape(name)
						name = nil if (dir+name).exist?
						break if name
					end
				}
			end
		rescue Exception
		end 
		#unique name?
		loop {
			name = Time.now.to_f.to_s+ext[0]
			break unless (dir+name).exist?
			sleep 1
		}unless name 
		dir+name
	end 
end
class OPML
	def initialize(title = nil)
		@doc = Document.new
		@doc.xml_decl.dowrite
		@doc.add_element Element.new("opml")
		@doc.root.add_attribute "version", "1.1"
		head = Element.new("head")
		@doc.root.add_element head
		if title
			titlee = Element.new("title")
			titlee.text = title
			head.add_element titlee
		end
		@body = Element.new("body")
		@doc.root.add_element @body
		@size = 0
	end
	def add(feedurl, text=nil)
		e = Element.new("outline")
		e.add_attribute("text", text) if text
		e.add_attribute "type", "link"
		e.add_attribute "url", feedurl
		@body.add_element e
		@size += 1
	end 
	def write()
		@doc.write $stdout, 0
	end
	def size()
		@size
	end 
end 

class Query
	def initialize(opt, query)
		@@ATOM_NS = Regexp.new '^http://purl.org/atom/ns#'
		@@ITUNES_NS = 'http://www.itunes.com/dtds/podcast-1.0.dtd'
		@opt = opt
		if query
			@query = query.downcase.split
			@query = nil if @query.size == 0
		end
	end
	def search(urls)
		res = []
		begin 
			newpaths = []
			dochistory = []
			paths = []
			if urls.size == 0
				$stderr.puts "Reading subscriptions from standard input" if @opt.verbose
				begin 
					xml = ""
					$stdin.each(){|e|
						xml += e
					}
					path = OpenStruct.new
					path.doc = Document.new(xml)
					if path.doc and path.doc.root
						path.relevance = 0
						newpaths << path
					end
				rescue Interrupt, SystemExit
					raise
				rescue Exception
					$stderr.puts "Error: unreadable subscriptions"
				end 
			else 
				newpaths = urls.uniq.collect(){|e|
					path = OpenStruct.new
					path.url = e
					path
				}
				newpaths = newpaths.collect(){|path|
					$stderr.puts "Fetching: #{path.url}" if @opt.verbose
					dochistory << path.url
					path.doc = fetchdoc(path)
					if path.doc
						path.relevance = 0
						path
					else 
						$stderr.puts "Skipping unreadable document" if @opt.verbose
						nil
					end
				}
				newpaths.compact!
			end
			loop {	
				break if @opt.feeds and res.size >= @opt.feeds
				begin
					newpaths.sort!(){|path1, path2|
						path2.relevance <=> path1.relevance
					}
					paths = newpaths + paths
					newpaths = []
					path = nil
					loop {
						path = paths.shift
						break unless path
						if path.doc
							break
						else
							if dochistory.detect{|e| e == path.url}
								$stderr.puts "Skipping duplicate: #{path.url}" if @opt.verbose
								next
							end
							$stderr.puts "Fetching: #{path.url}" if @opt.verbose
							dochistory << path.url
							path.doc = fetchdoc(path)
							if path.doc
								break
							end
							$stderr.puts "Error: skipping unreadable document"
						end
					}
					break unless path
					if path.doc.root.name == "opml"
						#doc relevance
						path.relevance += relevance_of(XPath.first(path.doc, "/opml/head/title/text()"))
						#outgoing links
						XPath.each(path.doc,"//outline"){|outline|
							url = outline.attributes["xmlUrl"]
							url = outline.attributes["url"] unless url
							next unless url
							begin 
								url = URI.parse(path.url).merge(url).to_s if path.url
							rescue Interrupt, SystemExit
								raise
							rescue Exception
							end
							newpath = OpenStruct.new
							newpath.url = url
							newpath.referrer = path.url
							#link relevance
							newpath.relevance = path.relevance
							XPath.each(outline, "ancestor-or-self::outline"){|e|
								newpath.relevance += relevance_of(e.attributes["text"])
							}
							#
							newpaths << newpath
						}
					elsif path.doc.root.name == "pcast"
						#outgoing links
						XPath.each(path.doc,"/pcast/channel"){|channel|
							link = XPath.first(channel, "link[@rel='feed']")
							next unless link
							url = link.attributes["href"]
							next unless url
							begin 
								url = URI.parse(path.url).merge(url).to_s if path.url
							rescue Interrupt, SystemExit
								raise
							rescue Exception
							end
							newpath = OpenStruct.new
							newpath.url = url
							newpath.referrer = path.url
							#link relevance
							newpath.relevance = path.relevance
							newpath.relevance += relevance_of(XPath.first(channel, "title/text()"))
							newpath.relevance += relevance_of(XPath.first(channel, "subtitle/text()"))
							#
							newpaths << newpath
						}
					elsif path.doc.root.namespace =~ @@ATOM_NS and path.url
						#doc relevance
						title = nil
						begin 
							XPath.each(path.doc.root,"/*/*"){|e|
								next unless e.namespace =~ @@ATOM_NS
								next unless e.name == "title" or e.name == "subtitle"
								title = e.text if e.name == "title"
								path.relevance += relevance_of(e.text)
							}
						rescue Interrupt, SystemExit
							raise
						rescue Exception
							#$stderr.puts "error: #{$!}"
						end 
						if path.relevance > 0
							$stderr.puts "Found: #{title} (relevance: #{path.relevance})" if @opt.verbose
							if title
								path.title = ""
								title.value.each(){|e3| #remove line breaks
									path.title+= e3.chomp+" "
								}
								path.title.strip!
							end
							res << path
						end
					elsif path.doc.root.name = "rss" and path.url
						#doc relevance
						title = XPath.first(path.doc, "//channel/title/text()")
						path.relevance += relevance_of(title)
						path.relevance += relevance_of(XPath.first(path.doc, "//channel/description/text()"))
						begin 
							XPath.each(path.doc.root,"//channel/*"){|e|
								next unless e.name == "category"
								if e.namespace == @@ITUNES_NS
									XPath.each(e, "descendant-or-self::*"){|e2|
										next unless e2.name == "category"
										path.relevance += relevance_of(e2.attributes["text"])
									}
								else 
									path.relevance += relevance_of(e.text) 
								end
							}
						rescue Interrupt, SystemExit
							raise
						rescue Exception
							#$stderr.puts "error: #{$!}"
						end 
						if path.relevance > 0
							$stderr.puts "Found: #{title} (relevance: #{path.relevance})" if @opt.verbose
							if title
								path.title = ""
								title.value.each(){|e3| #remove line breaks
									path.title+= e3.chomp+" "
								}
								path.title.strip!
							end
							res << path
						end
					end
				rescue Interrupt, SystemExit
					raise
				rescue Exception
					$stderr.puts "Error: skipping unreadable document"
				end 
			}
		rescue Interrupt, SystemExit
			$stderr.puts "Execution interrupted"
		rescue Exception
		end 
		result = nil
		while not result
			begin 
				res.sort!(){|path1, path2|
					path2.relevance <=> path1.relevance
				}
				opml = OPML.new "Search results for \"#{@query.collect(){|e| "#{e} "}}\""
				res.each(){|path|
					opml.add path.url, path.title
				}
				result = opml
			rescue Exception
			end 
		end
		result.write
		result
	end 
private
	def relevance_of(meta)
		return 0 unless meta
		unless meta.kind_of? String #Text todo: resolve entities
			meta = meta.value
		end
		meta = meta.downcase
		meta = meta.split
		res = 0
		@query.each(){|e|
			meta.each(){|e2|
				res += 1 if e2.index(e)
			}
		}
		res
	end 
	def fetchdoc(link)
		doc = ""
		1.upto(@opt.retries){|i|
			begin 
				if link.url =~ %r{^http:} or link.url =~ %r{^ftp:}
					if link.referrer and (link.referrer =~ %r{^http:} or link.referrer =~ %r{^ftp:})
						open(link.url, "User-Agent" => @opt.USER_AGENT, "Referer" => link.referrer){|f|
							break if f.content_type.index "audio/"
							break if f.content_type.index "video/"
							f.each_line(){|e|
								doc += e
							}
						}
					else 
						open(link.url, "User-Agent" => @opt.USER_AGENT){|f|
							break if f.content_type.index "audio/"
							break if f.content_type.index "video/"
							f.each_line(){|e|
								doc += e
							}
						}
					end
				else 
					open(link.url){|f|
						f.each_line(){|e|
							doc += e
						}
					}
				end
				break
			rescue Interrupt
			rescue SystemExit
				break
			rescue Exception
			end 
			$stderr.puts "Attempt #{i} aborted" if @opt.verbose
			doc = ""
			sleep 5
		}
		res = nil
		begin 
			res = Document.new doc
		rescue Exception
		end 
		res = nil unless res and res.root
		res
	end 
end 

if opt.function == :download
	cache = Cache.new opt
	cache.createplaylist ARGV
else 
	dir = Query.new opt, ARGV.shift
	dir.search ARGV
end

$stderr.puts "Done." if opt.verbose

